framework,version,device,op_name,kernel_source,gemm_dtype,m,n,k,latency
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,4096,16.763904571533203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,3584,14.730239868164062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,5120,20.84351921081543
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,8192,33.139713287353516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,2560,10.654720306396484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,2048,8.6179838180542
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,7168,29.086719512939453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,3072,12.692480087280273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,128,1.1325440406799316
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,512,2.515968084335327
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,256,1.58515202999115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,1024,4.54860782623291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,1536,6.593535900115967
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,10240,41.20064163208008
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,8192,8.3056640625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,10240,10.313728332519531
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,16384,16.42803192138672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,16384,65.6394271850586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,4096,4.20249605178833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,12288,12.409855842590332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,7168,7.258111953735352
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,5120,5.243904113769531
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,1024,1.1509759426116943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,3584,3.719167947769165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,2048,2.1719040870666504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,128,0.29286399483680725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,512,0.6410239934921265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,256,0.40755200386047363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,1536,1.6578559875488281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,2560,2.6757121086120605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,3072,3.18668794631958
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,8192,6.213632106781006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,7168,5.45689582824707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,65536,12288,49.364990234375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,10240,7.739391803741455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,16384,12.390399932861328
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,5120,3.920896053314209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,4096,3.156991958618164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,3584,2.776063919067383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,2048,1.6261119842529297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,12288,9.336832046508789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,3072,2.392064094543457
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,1024,0.8652799725532532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,1536,1.2472319602966309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,256,0.3102720081806183
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,2560,2.017280101776123
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,128,0.22118400037288666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,512,0.48230400681495667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,8192,5.232639789581299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,7168,4.588543891906738
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,10240,6.534143924713135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,4096,2.65830397605896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,5120,3.3024001121520996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,16384,10.378239631652832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,3584,2.341887950897217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,12288,7.831552028656006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,2048,1.371135950088501
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,2560,1.6936960220336914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,3072,2.0142080783843994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,512,0.40959998965263367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,128,0.187391996383667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,1024,0.7321599721908569
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,256,0.26009601354599
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,1536,1.0516480207443237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,8192,4.169727802276611
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,10240,5.165056228637695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,16384,8.225791931152344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,12288,6.2320637702941895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,7168,3.6382720470428467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,5120,2.619391918182373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,4096,2.1073920726776123
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,2048,1.0895359516143799
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,3584,1.8524160385131836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,2560,1.3445119857788086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,1024,0.5847039818763733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,512,0.3266560137271881
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,3072,1.6066559553146362
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,128,0.1525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,256,0.21094399690628052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,1536,0.834559977054596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,10240,4.620287895202637
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,16384,7.319551944732666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,12288,5.52243185043335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,8192,3.7099521160125732
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,16384,65536,65.35577392578125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,5120,2.3490560054779053
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,7168,3.254271984100342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,4096,1.8851840496063232
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,2048,0.9748479723930359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,2560,1.2031999826431274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,3584,1.6588799953460693
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,3072,1.4305280447006226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,512,0.29183998703956604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,1024,0.52019202709198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,128,0.13414399325847626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,256,0.3758080005645752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,1536,0.7464960217475891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,12288,65536,49.19807815551758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,8192,2.632704019546509
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,10240,3.2675840854644775
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,16384,5.199872016906738
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,12288,3.9127039909362793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,10240,65536,40.96102523803711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,7168,2.30297589302063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,5120,1.6588799953460693
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,4096,1.3352960348129272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,3584,1.1816960573196411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,2048,0.6912000179290771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,1536,0.5304319858551025
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,2560,0.8560640215873718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,3072,1.0158079862594604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,512,0.2099200040102005
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,1024,0.37273600697517395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,128,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,256,0.13619199395179749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,8192,65536,32.737281799316406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,8192,2.0848639011383057
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,10240,2.6019840240478516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,7168,1.8288639783859253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,5120,1.3158400058746338
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,16384,4.123648166656494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,4096,1.06496000289917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,12288,3.123199939727783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,2560,0.6789119839668274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,2048,0.5519359707832336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,3584,0.9349120259284973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,1536,0.4259839951992035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,256,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,3072,0.8069120049476624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,512,0.16998399794101715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,128,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,1024,0.2979840040206909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,8192,1.8626559972763062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,10240,2.32857608795166
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,7168,65536,28.783615112304688
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,16384,3.689471960067749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,12288,2.787328004837036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,7168,1.6353280544281006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,4096,0.9492480158805847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,5120,1.1847679615020752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,2048,0.49459201097488403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,3584,0.8355839848518372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,2560,0.6082559823989868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,3072,0.7239680290222168
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,1536,0.37990400195121765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,512,0.1525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,1024,0.2682879865169525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,128,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,256,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,8192,1.6209919452667236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,10240,2.021375894546509
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,5120,65536,20.586496353149414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,16384,3.180543899536133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,12288,2.401279926300049
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,7168,1.423359990119934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,5120,1.031167984008789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,4096,0.8253440260887146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,3584,0.7270399928092957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,3072,0.6287360191345215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,2560,0.5283839702606201
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,2048,0.43110400438308716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,128,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,256,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,512,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,1024,0.23244799673557281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,1536,0.3307519853115082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,8192,1.3199360370635986
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,10240,1.6414719820022583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,4096,65536,16.384000778198242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,12288,1.9722239971160889
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,16384,2.6101760864257812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,4096,0.6737920045852661
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,7168,1.1601920127868652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,5120,0.8366079926490784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,2048,0.35123199224472046
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,2560,0.4362240135669708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,3584,0.5959680080413818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,3072,0.5130239725112915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,1536,0.27136000990867615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,128,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,512,0.11264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,256,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,1024,0.19353599846363068
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,8192,1.0680320262908936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3584,65536,14.512127876281738
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,10240,1.3332480192184448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,12288,1.5841280221939087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,16384,2.0951039791107178
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,4096,0.5468159914016724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,5120,0.6768640279769897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,7168,0.9379839897155762
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,2048,0.28569599986076355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,2560,0.35123199224472046
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,1536,0.22118400037288666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,3584,0.48127999901771545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,3072,0.41676801443099976
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,1024,0.15564799308776855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,128,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,256,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,512,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,8192,0.8366079926490784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,3072,65536,12.578816413879395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,10240,1.0362880229949951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,12288,1.230847954750061
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,16384,1.6261119842529297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,4096,0.43212801218032837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,7168,0.7352319955825806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,5120,0.5386239886283875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,3584,0.3829759955406189
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,2048,0.22732800245285034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,2560,0.27955201268196106
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,1536,0.17612800002098083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,3072,0.32972800731658936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,256,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,512,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,128,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,1024,0.12595200538635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,8192,0.5560320019721985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,10240,0.6881279945373535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2560,65536,10.383359909057617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,12288,0.8151040077209473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,16384,1.0803200006484985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,4096,0.2887679934501648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,7168,0.4904960095882416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,5120,0.3563520014286041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,3584,0.2549760043621063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,2048,0.1525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,3072,0.22118400037288666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,2560,0.1884160041809082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,1536,0.12083200365304947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,1024,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,512,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,128,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,256,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,8192,0.2959359884262085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,10240,0.3604480028152466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,2048,65536,8.224767684936523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,12288,0.4259839951992035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,16384,0.5591040253639221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,7168,0.26009601354599
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,5120,0.19148799777030945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,3584,0.13619199395179749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,4096,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,3072,0.11980800330638885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,2048,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,1024,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,2560,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,1536,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,128,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,512,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,256,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,8192,0.16383999586105347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1536,65536,6.295551776885986
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,10240,0.1955839991569519
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,12288,0.23552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,16384,0.3041279911994934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,4096,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,5120,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,7168,0.14643199741840363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,3584,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,2560,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,1536,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,2048,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,1024,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,3072,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,128,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,256,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,512,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,1024,65536,4.140031814575195
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,8192,0.11264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,10240,0.13414399325847626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,12288,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,5120,0.07782399654388428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,16384,0.1955839991569519
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,4096,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,7168,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,3584,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,2560,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,2048,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,3072,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,1536,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,512,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,512,65536,2.2220799922943115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,8192,16.611328125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,5120,10.502143859863281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,256,65536,1.1868159770965576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8192,128,65536,0.7188479900360107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,4096,8.389632225036621
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,3584,7.406591892242432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,2048,4.3284478187561035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,2560,5.337088108062744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,3072,6.353919982910156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,256,0.7874559760093689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,512,1.2615679502487183
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,1024,2.2855679988861084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,128,0.5642240047454834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,7168,14.507007598876953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,1536,3.31059193611145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,65536,32.719871520996094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,8192,4.149248123168945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,10240,20.69913673400879
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,16384,8.265727996826172
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,10240,5.170176029205322
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,5120,2.6214399337768555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,12288,6.1829118728637695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,7168,3.6382720470428467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,4096,2.1176319122314453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,3584,1.8606079816818237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,2048,1.0885119438171387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,3072,1.610751986503601
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,512,0.3266560137271881
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,1024,0.591871976852417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,128,0.14950400590896606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,2560,1.3527040481567383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,256,0.20787200331687927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,16384,1536,0.8355839848518372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,16384,32.82944107055664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,5120,2.002943992614746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,4096,1.6138240098953247
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,8192,3.1682560443878174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,3072,1.222656011581421
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,7168,2.7781119346618652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,2048,0.8335360288619995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,10240,3.9577600955963135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,2560,1.0270719528198242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,3584,1.4161920547485352
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,256,0.159743994474411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,512,0.24985599517822266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,1024,0.44441598653793335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,128,0.11366400122642517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,16384,6.257664203643799
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,1536,0.638975977897644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,12288,4.737023830413818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,65536,12288,24.6824951171875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,8192,2.6255359649658203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,5120,1.6588799953460693
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,7168,2.303999900817871
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,10240,3.275775909423828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,4096,1.3363200426101685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,16384,5.223423957824707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,3584,1.1755520105361938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,2048,0.6922240257263184
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,2560,0.8560640215873718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,512,0.2088959962129593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,1024,0.3686400055885315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,12288,3.911679983139038
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,3072,1.0147839784622192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,256,0.13619199395179749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,128,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,1536,0.5345280170440674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,16384,4.121600151062012
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,8192,2.0848639011383057
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,7168,1.8391040563583374
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,5120,1.316864013671875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,3584,0.9390079975128174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,10240,2.6019840240478516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,4096,1.0618879795074463
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,2048,0.5529599785804749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,2560,0.6830080151557922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,65536,16.39321517944336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,3072,0.809984028339386
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,1536,0.4259839951992035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,512,0.16998399794101715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,128,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,1024,0.2959359884262085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,256,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,8192,12288,3.1283199787139893
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,8192,1.8626559972763062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,7168,1.644544005393982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,10240,2.3255040645599365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,4096,0.9492480158805847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,5120,1.1786240339279175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,16384,3.7027840614318848
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,12288,2.7750399112701416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,3584,0.8386560082435608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,1536,0.38092800974845886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,2048,0.49561598896980286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,256,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,2560,0.6082559823989868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,512,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,3072,0.7249919772148132
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,1024,0.2693119943141937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,128,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,8192,1.3271039724349976
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,10240,1.6476160287857056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,16384,2.60915207862854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,12288,1.9783680438995361
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,7168,1.1653120517730713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,5120,0.839680016040802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,4096,0.6737920045852661
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,2560,0.4352000057697296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,2048,0.35123199224472046
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,3584,0.596992015838623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,3072,0.5150719881057739
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,512,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,1024,0.18943999707698822
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,1536,0.2744320034980774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,256,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,128,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,12288,1.5953919887542725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,12288,65536,24.577024459838867
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,10240,1.32915198802948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,8192,1.0741759538650513
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,16384,2.1022720336914062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,7168,0.9390079975128174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,4096,0.549888014793396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,10240,65536,20.563968658447266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,5120,0.6778879761695862
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,3584,0.4843519926071167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,2048,0.28569599986076355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,2560,0.35225600004196167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,1536,0.22118400037288666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,1024,0.15462400019168854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,3072,0.41574400663375854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,512,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,256,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,128,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,10240,1.1970560550689697
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,8192,0.9584640264511108
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,7168,0.8447999954223633
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,5120,0.609279990196228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,16384,1.8810880184173584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,12288,1.4090240001678467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,4096,0.4915199875831604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,2560,0.31539198756217957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,3584,0.43212801218032837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,3072,0.374783992767334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,2048,0.25702399015426636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,512,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,256,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,1024,0.13926400244235992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,128,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,7168,65536,14.510080337524414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,1536,0.19865599274635315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,12288,1.2318719625473022
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,5120,65536,10.35366439819336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,8192,0.834559977054596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,4096,0.4331519901752472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,10240,1.0321919918060303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,5120,0.536575973033905
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,7168,0.7362560033798218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,16384,1.623039960861206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,2560,0.27852800488471985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,3584,0.38092800974845886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,2048,0.22630399465560913
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,3072,0.32972800731658936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,512,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,1024,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,128,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,256,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,1536,0.17715199291706085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,4096,65536,8.217599868774414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,8192,0.6676480174064636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3072,65536,6.323200225830078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,10240,0.8273919820785522
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,3584,0.3041279911994934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,4096,0.34508800506591797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,12288,0.9912319779396057
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,5120,0.4259839951992035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,16384,1.3137919902801514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,7168,0.5867519974708557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,1024,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,2560,0.22220799326896667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,512,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,2048,0.1812479943037033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,1536,0.14233599603176117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,3072,0.26214399933815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,256,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,128,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,10240,0.6860799789428711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,12288,0.8161280155181885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,8192,0.5550079941749573
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,7168,0.48947200179100037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,4096,0.2877439856529236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,3584,65536,7.373824119567871
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,5120,0.35737600922584534
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,3584,0.2549760043621063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,3072,0.22118400037288666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,2048,0.15462400019168854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,1024,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,2560,0.1884160041809082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,1536,0.12185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,16384,1.0751999616622925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,512,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,256,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,128,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,7168,0.3891200125217438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,8192,0.4403199851512909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,10240,0.5386239886283875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,4096,0.23244799673557281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,12288,0.6328319907188416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,5120,0.28569599986076355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,16384,0.8376320004463196
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,3584,0.20787200331687927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,2560,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,2048,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,1536,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,3072,0.17817600071430206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,1024,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,256,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,128,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,512,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,8192,0.29183998703956604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,10240,0.3604480028152466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,12288,0.42393600940704346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,16384,0.5550079941749573
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,7168,0.25804799795150757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,5120,0.18943999707698822
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2560,65536,5.189631938934326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,4096,0.1525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,3584,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,2048,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,2560,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,3072,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,1536,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,1024,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,512,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,256,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,128,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,8192,0.16076800227165222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,10240,0.19046400487422943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,12288,0.2314240038394928
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,2048,65536,4.136960029602051
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,16384,0.30105599761009216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,7168,0.14336000382900238
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,5120,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,4096,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,3584,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,3072,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,2048,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,1536,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,1024,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,2560,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,512,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,256,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1536,65536,3.270656108856201
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,8192,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,10240,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,7168,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,12288,0.12595200538635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,16384,0.159743994474411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,5120,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,4096,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,3584,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,3072,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,2048,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,2560,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,1024,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,1536,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,1024,65536,2.194432020187378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,7168,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,10240,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,8192,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,12288,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,16384,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,4096,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,5120,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,3584,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,2048,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,1536,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,512,65536,1.1089919805526733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,256,65536,0.6748160123825073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4096,128,65536,0.40755200386047363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,3584,3.6956160068511963
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,4096,4.20249605178833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,2560,2.6746881008148193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,5120,5.230591773986816
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,8192,8.316927909851074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,2048,2.165760040283203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,7168,7.260159969329834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,1024,1.1499520540237427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,3072,3.185663938522339
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,128,0.2887679934501648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,512,0.6369280219078064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,256,0.40038400888442993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,1536,1.6578559875488281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,8192,2.0848639011383057
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,10240,10.32089614868164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,10240,2.593791961669922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,16384,4.139008045196533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,5120,1.316864013671875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,3072,0.8069120049476624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,16384,16.424959182739258
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,12288,3.1006720066070557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,4096,1.060863971710205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,7168,1.8288639783859253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,2048,0.5509120225906372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,256,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,1024,0.2959359884262085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,128,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,512,0.16896000504493713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,3584,0.9379839897155762
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,2560,0.6799359917640686
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,1536,0.5724160075187683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,8192,1.6312320232391357
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,7168,1.423359990119934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,65536,12288,12.406784057617188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,4096,0.826367974281311
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,10240,2.02239990234375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,16384,3.1784958839416504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,5120,1.026047945022583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,3584,0.7270399928092957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,2560,0.532480001449585
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,12288,2.4033279418945312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,3072,0.6277120113372803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,2048,0.42905598878860474
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,256,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,512,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,128,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,1024,0.23244799673557281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,1536,0.3317759931087494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,7168,1.159168004989624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,16384,2.6112000942230225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,8192,1.3199360370635986
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,4096,0.6748160123825073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,3584,0.596992015838623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,5120,0.8366079926490784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,10240,1.6424959897994995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,2048,0.35225600004196167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,2560,0.4352000057697296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,3072,0.5140479803085327
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,65536,10.34444808959961
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,512,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,256,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,1024,0.19148799777030945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,128,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,1536,0.27238398790359497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,10240,12288,1.9752960205078125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,10240,1.3312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,12288,1.605631947517395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,5120,0.6778879761695862
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,4096,0.5468159914016724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,8192,1.0680320262908936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,3584,0.48230400681495667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,7168,0.9441279768943787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,2560,0.35123199224472046
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,2048,0.28467199206352234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,3072,0.41676801443099976
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,512,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,1536,0.22118400037288666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,1024,0.15667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,256,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,128,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,16384,2.096127986907959
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,8192,0.9564160108566284
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,7168,0.8407040238380432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,10240,1.1898880004882812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,5120,0.6072319746017456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,16384,1.8759679794311523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,12288,1.410048007965088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,4096,0.4904960095882416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,3584,0.43212801218032837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,2048,0.25702399015426636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,2560,0.31539198756217957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,3072,0.37376001477241516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,1024,0.14028799533843994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,128,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,256,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,512,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,1536,0.19865599274635315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,12288,0.9912319779396057
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,16384,1.3137919902801514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,16384,65536,16.490495681762695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,10240,0.8304640054702759
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,8192,0.6727679967880249
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,3584,0.3041279911994934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,7168,0.5867519974708557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,3072,0.26316800713539124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,4096,0.34303998947143555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,5120,0.4280320107936859
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,1024,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,2560,0.22323200106620789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,12288,65536,12.355584144592285
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,2048,0.1812479943037033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,1536,0.14131200313568115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,512,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,256,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,128,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,5120,65536,5.189631938934326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,8192,0.5560320019721985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,12288,0.8253440260887146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,5120,0.3563520014286041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,4096,0.2908160090446472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,7168,0.48844799399375916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,10240,0.6850559711456299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,3584,0.25600001215934753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,3072,0.22220799326896667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,2560,0.1884160041809082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,512,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,2048,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,1024,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,16384,1.0751999616622925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,1536,0.12083200365304947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,256,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,128,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,8192,0.5017600059509277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,8192,65536,8.255488395690918
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,4096,65536,4.137983798980713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,7168,0.4403199851512909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,10240,0.6154239773750305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,4096,0.26419198513031006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,5120,0.32256001234054565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,3584,0.23347200453281403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,2048,0.14131200313568115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,16384,0.9646080136299133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,12288,0.7229440212249756
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,1024,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,2560,0.17203199863433838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,3072,0.2027519941329956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,256,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,128,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,512,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,1536,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,7168,65536,7.390207767486572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,7168,0.38707199692726135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,5120,0.28672000765800476
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,8192,0.43827199935913086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,4096,0.23449599742889404
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,3584,0.20787200331687927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,2560,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,10240,0.536575973033905
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,2048,0.12595200538635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,3072,0.17919999361038208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,1024,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,12288,0.6318079829216003
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,1536,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,512,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,16384,0.834559977054596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,256,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,128,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,16384,0.6666240096092224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,8192,0.3420160114765167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,7168,0.30720001459121704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,3584,0.159743994474411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,10240,0.42291200160980225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,4096,0.17919999361038208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,3072,0.13926400244235992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,12288,0.5058559775352478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,5120,0.22118400037288666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,2560,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,512,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,2048,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,1024,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,128,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,1536,0.07782399654388428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,256,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,8192,0.29388800263404846
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,10240,0.35942399501800537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,7168,0.2590720057487488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,5120,0.18943999707698822
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,12288,0.42393600940704346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,4096,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,16384,0.5560320019721985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,3584,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,3072,0.11980800330638885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,2048,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,1536,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3584,65536,3.7171199321746826
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,2560,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,256,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,512,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,128,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,1024,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,10240,0.27852800488471985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,8192,0.23244799673557281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,16384,0.4413439929485321
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,3072,65536,3.26963210105896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,12288,0.32972800731658936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,4096,0.12390399724245071
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,7168,0.20582400262355804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,5120,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,3584,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,3072,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,1024,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,2560,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,2048,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,512,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,1536,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,128,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,256,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,8192,0.16076800227165222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,10240,0.19046400487422943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2560,65536,2.6163198947906494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,5120,0.31334400177001953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,7168,0.14131200313568115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,12288,0.2303999960422516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,4096,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,16384,0.30003198981285095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,3584,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,1024,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,2048,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,3072,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,2560,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,1536,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,512,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,2048,65536,2.1882879734039307
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,8192,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,10240,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,12288,0.12390399724245071
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,7168,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,16384,0.159743994474411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,4096,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,5120,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,3584,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,2560,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,3072,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,1536,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,2048,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1536,65536,1.7489919662475586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,10240,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,8192,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,7168,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,12288,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,5120,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,4096,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,16384,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,3584,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,3072,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,1024,65536,1.1151360273361206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,12288,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,8192,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,7168,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,10240,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,4096,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,5120,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,16384,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,3584,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,3072,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,512,65536,0.6758400201797485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,2560,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,2048,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,10240,5.169151782989502
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,256,65536,0.3973119854927063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2048,128,65536,0.24883200228214264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,4096,2.1135358810424805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,5120,2.619391918182373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,3584,1.8554879426956177
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,2048,1.0895359516143799
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,1024,0.5806080102920532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,512,0.3246079981327057
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,8192,4.149248123168945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,128,0.14950400590896606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,256,0.20787200331687927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,7168,3.644416093826294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,3072,1.6076799631118774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,2560,1.346560001373291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,1536,0.8355839848518372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,7168,0.936959981918335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,8192,1.0690560340881348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,65536,8.286208152770996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,10240,1.3332480192184448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,5120,0.6748160123825073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,3584,0.4843519926071167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,4096,0.5457919836044312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,16384,2.0951039791107178
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,2048,0.28467199206352234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,12288,1.5861760377883911
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,1024,0.15667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,2560,0.35123199224472046
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,128,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,1536,0.22220799326896667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,256,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,512,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,16384,3072,0.41574400663375854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,16384,8.261631965637207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,12288,1.2267520427703857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,5120,0.536575973033905
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,4096,0.4331519901752472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,8192,0.834559977054596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,7168,0.7331839799880981
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,2048,0.22732800245285034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,65536,12288,6.18393611907959
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,10240,1.031167984008789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,2560,0.27955201268196106
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,3584,0.3840000033378601
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,3072,0.3307519853115082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,256,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,512,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,128,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,1024,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,16384,1.6220159530639648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,1536,0.17715199291706085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,12288,65536,6.295551776885986
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,8192,0.6696959733963013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,10240,0.8325120210647583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,7168,0.5867519974708557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,4096,0.34406399726867676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,16384,1.3230079412460327
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,2048,0.18329599499702454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,1536,0.14336000382900238
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,3584,0.3041279911994934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,1024,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,5120,0.4331519901752472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,512,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,3072,0.26419198513031006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,2560,0.22323200106620789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,256,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,128,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,12288,0.9912319779396057
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,8192,0.5529599785804749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,10240,65536,5.201920032501221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,7168,0.48742398619651794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,10240,0.6830080151557922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,4096,0.2887679934501648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,2560,0.1884160041809082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,5120,0.3563520014286041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,3584,0.2549760043621063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,2048,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,12288,0.8171520233154297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,3072,0.22323200106620789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,16384,1.0833920240402222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,1536,0.12083200365304947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,256,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,512,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,128,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,1024,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,8192,65536,4.154367923736572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,5120,0.32256001234054565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,12288,0.7219200134277344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,10240,0.6123520135879517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,16384,0.9809920191764832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,3584,0.23347200453281403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,3072,0.2027519941329956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,2560,0.17203199863433838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,4096,0.26419198513031006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,1536,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,2048,0.13926400244235992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,512,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,1024,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,128,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,8192,0.4976640045642853
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,256,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,7168,0.4413439929485321
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,5120,0.22118400037288666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,8192,0.3420160114765167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,10240,0.42291200160980225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,3584,0.158720001578331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,4096,0.17919999361038208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,7168,0.30105599761009216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,3072,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,2560,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,2048,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,1536,0.07782399654388428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,1024,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,512,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,7168,65536,3.7007360458374023
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,16384,0.6686720252037048
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,256,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,12288,0.5038080215454102
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,128,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,5120,0.187391996383667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,8192,0.29286399483680725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,7168,0.25702399015426636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,3584,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,10240,0.35839998722076416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,4096,0.1525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,2048,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,12288,0.42393600940704346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,16384,0.5529599785804749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,3072,0.11673600226640701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,2560,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,1536,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,1024,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,128,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,512,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,256,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,12288,0.37785598635673523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,16384,0.4976640045642853
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,8192,0.26214399933815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,7168,0.23347200453281403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,4096,0.1454080045223236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,5120,0.17510400712490082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,10240,0.32051199674606323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,3584,0.130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,1536,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,2560,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,2048,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,3072,0.11468800157308578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,1024,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,256,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,128,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,512,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,8192,0.2314240038394928
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,12288,0.32870399951934814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,16384,0.43929600715637207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3584,65536,1.977344036102295
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,7168,0.20377600193023682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,3584,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,10240,0.2764799892902374
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,4096,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,5120,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,1024,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,2048,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,3072,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,5120,65536,2.6357760429382324
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,2560,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,512,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,1536,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,256,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,128,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,8192,0.17919999361038208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,12288,0.2672640085220337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,7168,0.16076800227165222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,10240,0.22732800245285034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,3072,65536,1.7500159740447998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,4096,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,4096,65536,2.1872639656066895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,3584,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,5120,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,3072,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,16384,0.3491840064525604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,2560,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,2048,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,1024,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,512,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,1536,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,5120,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,12288,0.22835199534893036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,8192,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,7168,0.14233599603176117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,4096,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,10240,0.18943999707698822
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,3584,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,1536,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,2560,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,2048,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,3072,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,16384,0.2969599962234497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,512,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,1024,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,12288,0.17612800002098083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,16384,0.23654399812221527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,7168,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,8192,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,10240,0.14745600521564484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,4096,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,5120,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,2048,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,3072,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,2560,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,3584,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,1024,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,1536,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2560,65536,1.3322240114212036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,10240,0.10342399775981903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,8192,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,5120,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,12288,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,7168,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,4096,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,16384,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,3584,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,3072,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,1024,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,2048,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,1536,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,2560,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,2048,65536,1.1079679727554321
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,8192,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,7168,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,10240,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,12288,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,5120,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,4096,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1536,65536,0.8908799886703491
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,16384,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,1024,65536,0.6748160123825073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,3584,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,3072,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,2560,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,2048,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,7168,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,5120,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,8192,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,10240,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,12288,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,16384,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,4096,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,7168,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,8192,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,10240,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,12288,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,512,65536,0.4331519901752472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,16384,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,5120,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,256,65536,0.23756800591945648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,3584,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,4096,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,3072,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,7168,2.7781119346618652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,12288,4.7226881980896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,3584,1.4161920547485352
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,16384,6.301695823669434
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,2560,1.0280959606170654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,1536,0.638975977897644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1024,128,65536,0.14336000382900238
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,512,0.24883200228214264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,256,0.16076800227165222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,128,0.11468800157308578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,4096,1.6179200410842896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,2048,0.8366079926490784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,1024,0.44543999433517456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,12288,1.257472038269043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,10240,1.0496000051498413
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,16384,1.6588799953460693
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,7168,0.7403519749641418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,5120,0.5345280170440674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,4096,0.43212801218032837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,3584,0.3840000033378601
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,3072,1.222656011581421
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,5120,2.001919984817505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,2560,0.27955201268196106
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,1536,0.17612800002098083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,1024,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,8192,0.8468480110168457
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,512,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,2048,0.22732800245285034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,256,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,128,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,8192,3.167232036590576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,3072,0.32972800731658936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,10240,0.7905279994010925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,12288,0.9400320053100586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,8192,0.6369280219078064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,7168,0.5601279735565186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,5120,0.3973119854927063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,2560,0.2088959962129593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,4096,0.3235839903354645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,3584,0.2826240062713623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,3072,0.24473600089550018
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,2048,0.16998399794101715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,1024,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,128,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,512,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,256,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,1536,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,12288,0.8314880132675171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,16384,1.227776050567627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,65536,10240,3.945472002029419
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,8192,0.5601279735565186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,5120,0.35737600922584534
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,10240,0.6932479739189148
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,3584,0.25702399015426636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,7168,0.4915199875831604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,16384,1.0977280139923096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,2560,0.1884160041809082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,2048,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,65536,4.330495834350586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,512,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,4096,0.2908160090446472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,256,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,1024,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,128,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,3072,0.22220799326896667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,10240,1536,0.12083200365304947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,7168,0.38809600472450256
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,8192,0.4413439929485321
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,5120,0.2877439856529236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,10240,0.5416960120201111
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,16384,0.8499199748039246
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,4096,0.23347200453281403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,12288,0.6451200246810913
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,2560,0.1525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,3584,0.20684799551963806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,3072,0.1802240014076233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,2048,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,1024,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,512,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,256,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,1536,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,128,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,8192,0.3840000033378601
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,10240,0.47513601183891296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,16384,0.7403519749641418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,12288,0.5529599785804749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,3584,0.18329599499702454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,7168,0.3389439880847931
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,5120,0.2519040107727051
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,4096,0.20479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,2560,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,2048,0.11264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,3072,0.159743994474411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,1536,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,512,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,1024,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,128,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,256,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,10240,0.42188799381256104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,12288,65536,4.783103942871094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,8192,0.3399679958820343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,16384,0.6666240096092224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,12288,0.5038080215454102
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,16384,65536,6.487040042877197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,7168,0.30003198981285095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,5120,0.22015999257564545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,4096,0.19046400487422943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,2560,0.11673600226640701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,3584,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,3072,0.1382399946451187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,8192,65536,3.274751901626587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,2048,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,256,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,512,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,1024,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,128,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,1536,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,8192,0.23654399812221527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,12288,0.33792001008987427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,10240,0.28467199206352234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,16384,0.43929600715637207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,3072,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,7168,0.2088959962129593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,4096,0.12390399724245071
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,3584,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,5120,0.1525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,512,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,2560,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,1536,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,2048,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,1024,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,128,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,256,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,7168,65536,2.835455894470215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,10240,0.24780799448490143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,4096,65536,1.7541120052337646
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,8192,0.20479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,4096,0.11468800157308578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,5120,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,12288,0.2908160090446472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,7168,0.18432000279426575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,3584,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,16384,0.38604798913002014
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,1536,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,2560,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,1024,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,512,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,2048,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,3072,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,128,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,256,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,5120,65536,2.6224639415740967
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,8192,0.2314240038394928
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,5120,0.14950400590896606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,4096,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,7168,0.2027519941329956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,3072,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,10240,0.2181120067834854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,3584,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,2560,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,12288,0.24371199309825897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,1024,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,2048,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,16384,0.32767999172210693
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,512,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,1536,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,128,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,8192,0.17919999361038208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,7168,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,10240,0.22118400037288666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,4096,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,12288,0.26419198513031006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,16384,0.347135990858078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,5120,0.11673600226640701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,2560,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,3584,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,1536,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,1024,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,3072,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,2048,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,256,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,512,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,8192,0.130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3584,65536,1.5380480289459229
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,7168,0.1157120019197464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,10240,0.14643199741840363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,12288,0.18534399569034576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,5120,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,4096,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,16384,0.2519040107727051
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,3584,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,2048,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,3072,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,2560,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,1536,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,3072,65536,1.7500159740447998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,8192,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2048,65536,0.907263994216919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,10240,0.11673600226640701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,7168,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,4096,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,12288,0.13619199395179749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,3584,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,5120,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,3072,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,16384,0.23654399812221527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,1024,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,512,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,2048,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,2560,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,1536,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,2560,65536,1.106943964958191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,10240,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,7168,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,16384,0.12902399897575378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,5120,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,8192,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1536,65536,0.8888319730758667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,3072,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,3584,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,4096,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,12288,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,2560,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,1024,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,1536,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,12288,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,5120,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,7168,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,8192,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,10240,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,4096,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,16384,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,1024,65536,0.46592000126838684
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,3584,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,3072,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,12288,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,8192,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,10240,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,5120,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,16384,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,7168,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,4096,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,3584,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,3072,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,512,65536,0.3338240087032318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,5120,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,10240,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,7168,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,8192,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,16384,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,12288,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,4096,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,3584,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,12288,3.102720022201538
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,256,65536,0.14131200313568115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,16384,4.143104076385498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,3584,0.9410560131072998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,4096,1.0618879795074463
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,768,128,65536,0.1382399946451187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,1536,0.4249599874019623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,2048,0.5519359707832336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,1024,0.2949120104312897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,5120,1.3158400058746338
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,128,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,512,0.16793599724769592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,2560,0.6850559711456299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,256,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,8192,2.0848639011383057
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,3072,0.809984028339386
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,7168,1.8268159627914429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,5120,0.35839998722076416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,8192,0.5570560097694397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,7168,0.4904960095882416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,4096,0.2887679934501648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,10240,0.6850559711456299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,2048,0.15462400019168854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,65536,10240,2.5917439460754395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,3584,0.2549760043621063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,16384,1.0751999616622925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,2560,0.187391996383667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,1024,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,12288,0.8181759715080261
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,3072,0.22323200106620789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,128,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,512,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,256,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,1536,0.12083200365304947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,5120,0.28467199206352234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,8192,0.43827199935913086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,7168,0.38809600472450256
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,4096,0.23244799673557281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,10240,0.5416960120201111
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,3072,0.17919999361038208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,2048,0.12595200538635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,1536,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,16384,0.8591359853744507
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,12288,0.634880006313324
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,2560,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,3584,0.2088959962129593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,1024,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,256,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,128,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,512,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,8192,0.34303998947143555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,7168,0.3020800054073334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,10240,0.4259839951992035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,16384,0.6666240096092224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,4096,0.17817600071430206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,5120,0.22015999257564545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,3584,0.158720001578331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,3072,0.13926400244235992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,1536,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,1024,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,12288,0.5038080215454102
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,2048,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,2560,0.11878400295972824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,512,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,128,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,256,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,12288,0.4270080029964447
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,8192,0.29183998703956604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,7168,0.2590720057487488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,5120,0.18943999707698822
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,3584,0.13619199395179749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,4096,0.15462400019168854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,10240,0.35839998722076416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,3072,0.11878400295972824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,2560,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,16384,0.5539839863777161
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,2048,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,1536,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,512,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,1024,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,256,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,128,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,8192,65536,2.195456027984619
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,16384,0.5007359981536865
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,16384,65536,4.134912014007568
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,7168,0.23449599742889404
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,8192,0.26419198513031006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,10240,0.32153600454330444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,3584,0.131071999669075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,12288,0.37887999415397644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,4096,0.1454080045223236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,5120,0.1740799993276596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,2048,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,1024,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,3072,0.1157120019197464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,2560,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,512,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,12288,65536,3.26963210105896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,1536,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,128,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,256,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,12288,0.26419198513031006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,5120,0.11980800330638885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,7168,0.159743994474411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,8192,0.1812479943037033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,10240,0.22220799326896667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,4096,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,16384,0.347135990858078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,10240,65536,2.6224639415740967
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,3584,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,1536,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,3072,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,1024,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,2048,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,2560,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,512,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,12288,0.2314240038394928
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,7168,0.14336000382900238
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,10240,0.18943999707698822
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,8192,0.159743994474411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,16384,0.3041279911994934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,4096,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,3072,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,2560,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,3584,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,5120,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,2048,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,1536,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,1024,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,512,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,12288,0.20377600193023682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,10240,0.17510400712490082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,8192,0.14643199741840363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,7168,0.131071999669075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,4096,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,5120,0.12185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,4096,65536,1.1110399961471558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,7168,65536,1.9865599870681763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,16384,0.27750399708747864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,3584,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,3072,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,2560,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,2048,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,512,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,1024,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,1536,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,5120,65536,1.3322240114212036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,5120,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3584,65536,1.1079679727554321
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,8192,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,7168,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,12288,0.1740799993276596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,3584,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,10240,0.1525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,4096,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,16384,0.23552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,2048,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,1536,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,3072,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,2560,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,1024,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,5120,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,7168,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,8192,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,3584,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,4096,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,10240,0.12390399724245071
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,12288,0.1454080045223236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,2048,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,16384,0.187391996383667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,1024,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,1536,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,3072,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,2560,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,8192,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,16384,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2560,65536,0.7229440212249756
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,7168,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,10240,0.10342399775981903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,12288,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,5120,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,2048,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,4096,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,3584,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,3072,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,3072,65536,0.8898559808731079
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,1536,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,2560,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,1024,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,8192,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,5120,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,7168,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,10240,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,4096,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,3584,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,12288,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,2048,65536,0.6768640279769897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,3072,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,2048,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,16384,0.131071999669075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,2560,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,1536,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,1024,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,12288,0.07782399654388428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,5120,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,7168,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,16384,0.10342399775981903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,8192,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,4096,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,3584,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,3072,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,10240,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,16384,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,7168,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,8192,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,12288,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,10240,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,4096,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,5120,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,3584,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1536,65536,0.4710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,8192,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,12288,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,10240,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,7168,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,5120,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,16384,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,4096,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,3584,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,1024,65536,0.4710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,3072,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,1024,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,512,65536,0.2396160066127777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,8192,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,16384,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,12288,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,7168,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,10240,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,4096,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,256,65536,0.13926400244235992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,5120,1.0414079427719116
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,512,128,65536,0.11366400122642517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,10240,2.0480000972747803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,7168,1.44486403465271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,12288,2.450432062149048
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,8192,1.6476160287857056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,16384,3.255295991897583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,1536,0.33484798669815063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,256,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,128,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,1024,0.23449599742889404
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,512,0.13414399325847626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,2048,0.43724799156188965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,4096,0.8427519798278809
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,12288,0.6440960168838501
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,2560,0.5386239886283875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,16384,0.8540160059928894
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,8192,0.4423680007457733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,10240,0.5416960120201111
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,7168,0.38809600472450256
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,5120,0.2887679934501648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,3584,0.20684799551963806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,3072,0.1802240014076233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,3584,0.7393280267715454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,1024,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,65536,3072,0.638975977897644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,512,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,2048,0.130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,2560,0.1525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,4096,0.23347200453281403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,128,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,1536,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,256,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,10240,0.42393600940704346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,8192,0.3389439880847931
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,16384,0.6369280219078064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,3584,0.159743994474411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,4096,0.17919999361038208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,5120,0.21503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,7168,0.29286399483680725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,1536,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,3072,0.1382399946451187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,2048,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,512,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,2560,0.11878400295972824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,1024,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,128,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,256,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,12288,0.46592000126838684
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,12288,0.506879985332489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,8192,0.3420160114765167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,10240,0.42393600940704346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,5120,0.22015999257564545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,3584,0.158720001578331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,12288,65536,2.406399965286255
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,3072,0.13926400244235992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,4096,0.17715199291706085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,2048,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,1024,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,7168,0.30003198981285095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,2560,0.11673600226640701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,128,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,512,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,256,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,1536,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,16384,0.6656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,12288,0.33587199449539185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,10240,0.28569599986076355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,16384,0.43929600715637207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,3584,0.11264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,8192,0.2396160066127777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,4096,0.12595200538635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,7168,0.21196800470352173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,5120,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,2048,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,1536,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,2560,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,3072,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,1024,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,512,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,256,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,128,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,12288,0.29388800263404846
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,8192,0.20582400262355804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,10240,0.24780799448490143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,4096,0.1157120019197464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,5120,0.13619199395179749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,3584,0.10342399775981903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,7168,0.18227200210094452
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,16384,0.3850240111351013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,2048,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,3072,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,1536,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,2560,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,1024,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,65536,1.5411200523376465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,512,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,128,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,7168,256,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,8192,0.17919999361038208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,10240,0.22220799326896667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,16384,0.347135990858078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,5120,0.11673600226640701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,7168,0.159743994474411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,4096,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,12288,0.26419198513031006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,3072,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,2048,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,16384,65536,3.26963210105896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,2560,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,3584,0.2529279887676239
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,256,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,1536,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,1024,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,512,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,5120,65536,1.1130880117416382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,10240,65536,2.614272117614746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,7168,0.11878400295972824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,5120,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,16384,0.24371199309825897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,4096,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,3584,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,10240,0.16179199516773224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,8192,0.131071999669075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,8192,65536,1.7582080364227295
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,12288,0.18636800348758698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,2560,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,1024,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,3072,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,1536,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,2048,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,12288,0.159743994474411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,5120,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,10240,0.13209599256515503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,16384,0.2181120067834854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,4096,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,8192,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,7168,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,3584,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,2560,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,2048,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,3072,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,1024,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,1536,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,12288,0.131071999669075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,8192,0.12390399724245071
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,7168,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,10240,0.11673600226640701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,16384,0.23449599742889404
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3584,65536,0.8929280042648315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,3584,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,4096,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,2560,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,2048,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,3072,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,5120,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,1024,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,1536,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,12288,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,4096,65536,0.9093120098114014
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,16384,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,5120,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,8192,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,4096,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,7168,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,3072,65536,0.8898559808731079
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,3584,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,10240,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,3072,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,2048,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,2560,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,1024,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,1536,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,7168,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,16384,0.130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,8192,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,5120,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,3584,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,12288,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,3072,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2560,65536,0.7147520184516907
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,4096,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,10240,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,2560,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,2048,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,128,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,12288,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,5120,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,7168,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,3584,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,10240,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,8192,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,4096,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,16384,0.12902399897575378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,2048,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,2560,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,1536,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,1024,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,3072,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,12288,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,16384,0.07782399654388428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,8192,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,7168,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1536,65536,0.4628480076789856
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,4096,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,10240,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,5120,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,3584,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,2048,65536,0.4638719856739044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,2560,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,3072,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,10240,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,8192,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,5120,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,12288,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,7168,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,4096,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,16384,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,3584,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,1024,65536,0.2959359884262085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,3072,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,12288,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,8192,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,10240,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,7168,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,512,65536,0.1382399946451187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,5120,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,16384,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,3584,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,4096,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,7168,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,10240,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,5120,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,8192,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,12288,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,16384,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,4096,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,256,65536,0.14336000382900238
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,7168,0.9400320053100586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,8192,1.0782719850540161
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,4096,0.5509120225906372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,384,128,65536,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,2560,0.3532800078392029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,3584,0.4843519926071167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,3072,0.4208639860153198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,16384,2.106368064880371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,512,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,256,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,1536,0.22323200106620789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,128,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,1024,0.15564799308776855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,2048,0.28569599986076355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,5120,0.6778879761695862
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,12288,0.4280320107936859
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,16384,0.5580800175666809
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,10240,0.35942399501800537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,7168,0.2590720057487488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,4096,0.15462400019168854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,5120,0.19046400487422943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,3072,0.11980800330638885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,2048,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,2560,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,8192,0.29388800263404846
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,1536,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,256,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,512,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,3584,0.14847999811172485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,1024,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,128,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,10240,1.3332480192184448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,12288,0.33792001008987427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,10240,0.2805759906768799
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,8192,0.2314240038394928
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,16384,0.4403199851512909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,7168,0.20377600193023682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,3072,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,3584,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,5120,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,4096,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,2560,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,2048,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,1024,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,1536,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,256,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,128,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,512,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,12288,0.26521599292755127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,65536,12288,1.5872000455856323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,10240,0.2242559939622879
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,16384,0.3491840064525604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,4096,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,7168,0.16076800227165222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,8192,0.18329599499702454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,5120,0.11878400295972824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,3072,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,2048,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,3584,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,2560,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,1536,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,1024,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,512,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,8192,0.16896000504493713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,10240,0.19148799777030945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,12288,0.22937600314617157
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,5120,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,7168,0.14233599603176117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,4096,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,16384,0.30003198981285095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,10240,65536,1.3250559568405151
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,2048,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,3072,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,2560,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,3584,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,1536,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,512,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,1024,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,16384,65536,2.194432020187378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,12288,0.20377600193023682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,10240,0.17612800002098083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,8192,0.14847999811172485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,12288,65536,1.7489919662475586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,16384,0.27238398790359497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,7168,0.131071999669075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,4096,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,5120,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,3584,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,2048,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,2560,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,1024,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,3072,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,512,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,1536,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,7168,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,16384,0.1884160041809082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,8192,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,10240,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,4096,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,5120,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,12288,0.14643199741840363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,3584,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,7168,65536,1.1079679727554321
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,2048,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,2560,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,1536,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,3072,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,1024,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,7168,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,10240,0.10342399775981903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,12288,0.12185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,8192,65536,1.1089919805526733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,4096,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,3584,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,8192,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,5120,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,16384,0.158720001578331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,2560,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,5120,65536,0.7055360078811646
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,1024,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,3072,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,1536,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,2048,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,5120,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,8192,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,7168,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,4096,65536,0.6696959733963013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,12288,0.12595200538635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,16384,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,3072,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,3584,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,10240,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,4096,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,2048,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,1024,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,2560,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,1536,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,10240,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,7168,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,5120,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,8192,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,12288,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,4096,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,16384,0.12697599828243256
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,3584,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,3072,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,12288,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3072,65536,0.46489599347114563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,8192,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,10240,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,16384,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,7168,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,4096,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,5120,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,3584,65536,0.6154239773750305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,2048,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,3072,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,3584,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,5120,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,7168,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,8192,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,10240,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2560,65536,0.47308799624443054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,12288,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,4096,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,16384,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,3584,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,3072,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,2048,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,8192,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,10240,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,5120,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,7168,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,16384,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,4096,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,12288,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,2560,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,2048,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,3584,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,2048,65536,0.40755200386047363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,3072,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,8192,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,5120,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,10240,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,7168,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,16384,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,4096,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,12288,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,3072,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,2560,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1536,65536,0.2590720057487488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,3584,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,12288,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,10240,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,8192,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,5120,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,16384,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,7168,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,4096,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,2560,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,3584,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,7168,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,16384,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,8192,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,12288,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,512,65536,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,4096,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,10240,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,5120,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,1024,65536,0.2396160066127777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,12288,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,16384,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,10240,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,5120,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,7168,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,256,65536,0.11878400295972824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,256,128,65536,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,5120,0.5386239886283875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,8192,0.8509439826011658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,12288,1.581055998802185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,10240,1.3352960348129272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,3584,0.3840000033378601
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,3072,0.3317759931087494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,2560,0.2815999984741211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,2048,0.22732800245285034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,128,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,1024,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,512,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,256,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,4096,0.4341759979724884
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,8192,0.24473600089550018
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,1536,0.17715199291706085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,10240,0.2887679934501648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,16384,0.4413439929485321
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,4096,0.12697599828243256
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,7168,0.21299199759960175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,3584,0.11264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,3072,0.11366400122642517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,2048,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,7168,0.8181759715080261
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,5120,0.15667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,2560,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,1024,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,256,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,512,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,12288,0.3389439880847931
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,1536,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,128,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,7168,0.20377600193023682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,16384,0.3317759931087494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,8192,0.2303999960422516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,5120,0.14950400590896606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,16384,65536,2.184191942214966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,2560,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,3584,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,10240,0.22015999257564545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,2048,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,4096,0.12390399724245071
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,1536,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,12288,0.2467840015888214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,3072,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,1024,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,512,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,128,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,256,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,65536,16384,2.0971519947052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,8192,0.1802240014076233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,5120,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,16384,0.3481599986553192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,4096,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,3072,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,7168,0.16076800227165222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,10240,0.2252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,3584,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,2560,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,2048,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,1536,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,12288,0.26316800713539124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,1024,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,512,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,8192,0.13209599256515503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,7168,0.11878400295972824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,10240,0.14745600521564484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,5120,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,12288,0.18943999707698822
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,16384,0.3041279911994934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,4096,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,3584,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,2560,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,3072,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,1024,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,1536,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,512,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,2048,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,8192,65536,1.1141120195388794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,10240,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,16384,0.21913599967956543
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,7168,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,8192,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,5120,0.07782399654388428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,4096,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,12288,65536,1.7541120052337646
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,3584,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,3072,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,12288,0.16076800227165222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,2048,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,2560,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,1536,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,12288,0.12390399724245071
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,16384,0.19251200556755066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,8192,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,10240,65536,1.3230079412460327
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,5120,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,7168,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,10240,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,3584,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,4096,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,3072,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,1024,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,2560,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,1536,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,2048,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,8192,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,7168,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,5120,65536,0.6912000179290771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,3584,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,10240,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,12288,0.10342399775981903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,4096,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,5120,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,16384,0.131071999669075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,3072,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,1536,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,2048,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,2560,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,7168,65536,1.1059199571609497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,12288,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,8192,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,7168,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,10240,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,16384,0.130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,5120,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,3072,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,3584,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,4096,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,2560,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,1536,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,2048,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,12288,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,8192,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,16384,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,7168,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,4096,65536,0.5242879986763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,5120,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,10240,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,4096,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,3584,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,3072,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,10240,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,16384,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,7168,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,8192,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,5120,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,12288,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,4096,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3584,65536,0.47308799624443054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,3072,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,3584,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,3072,65536,0.4689919948577881
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,16384,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,12288,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,8192,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,4096,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,7168,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,5120,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,3072,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,3584,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,10240,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,2560,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,1536,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2560,65536,0.36556801199913025
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,12288,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,7168,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,10240,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,5120,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,4096,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,8192,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,2560,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,16384,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,3072,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,3584,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,12288,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,2048,65536,0.29286399483680725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,4096,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,8192,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,7168,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,10240,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,5120,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,16384,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,3584,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,2560,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,3072,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,12288,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,10240,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,8192,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,16384,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1536,65536,0.2590720057487488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,7168,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,1024,65536,0.1382399946451187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,4096,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,5120,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,3584,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,12288,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,10240,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,16384,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,4096,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,12288,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,10240,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,16384,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,3584,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,512,65536,0.1382399946451187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,256,65536,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,192,128,65536,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,4096,0.4352000057697296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,5120,0.5375999808311462
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,7168,0.7444480061531067
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,3072,0.3317759931087494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,10240,1.3301759958267212
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,2560,0.27955201268196106
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,2048,0.22732800245285034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,1536,0.17510400712490082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,1024,0.12390399724245071
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,512,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,256,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,128,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,3584,0.3819519877433777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,12288,0.33792001008987427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,8192,0.23756800591945648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,8192,0.8458240032196045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,10240,0.2887679934501648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,5120,0.15564799308776855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,4096,0.12595200538635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,3584,0.11264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,7168,0.21094399690628052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,16384,0.4403199851512909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,2560,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,2048,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,1536,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,1024,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,256,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,512,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,128,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,3072,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,12288,0.2467840015888214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,8192,0.2303999960422516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,16384,0.3307519853115082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,7168,0.20377600193023682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,4096,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,16384,2.10534405708313
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,10240,0.27750399708747864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,5120,0.14950400590896606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,2560,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,3584,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,2048,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,1536,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,1024,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,512,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,3072,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,65536,12288,1.5820800065994263
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,256,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,128,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,12288,0.26419198513031006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,7168,0.16076800227165222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,10240,0.22015999257564545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,16384,0.347135990858078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,5120,0.11878400295972824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,12288,65536,1.751039981842041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,8192,0.1812479943037033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,3072,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,3584,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,2560,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,4096,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,2048,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,1536,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,1024,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,512,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,8192,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,7168,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,10240,0.14643199741840363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,16384,0.2457599937915802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,3584,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,4096,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,2560,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,5120,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,3072,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,12288,0.2303999960422516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,2048,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,1024,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,1536,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,16384,65536,2.1882879734039307
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,10240,0.13209599256515503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,8192,0.11673600226640701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,7168,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,4096,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,12288,0.16076800227165222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,16384,0.21299199759960175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,5120,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,3072,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,2560,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,3584,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,2048,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,1536,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,1024,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,7168,65536,1.1079679727554321
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,12288,0.12390399724245071
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,8192,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,10240,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,10240,65536,1.3281279802322388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,7168,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,16384,0.18534399569034576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,5120,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,4096,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,2560,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,3072,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,3584,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,2048,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,8192,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,12288,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,5120,65536,0.6891520023345947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,16384,0.12902399897575378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,8192,65536,1.1079679727554321
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,10240,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,5120,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,3584,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,4096,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,7168,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,2560,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,2048,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,3072,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,8192,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,7168,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,10240,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,4096,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,5120,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,12288,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,16384,0.12902399897575378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,3072,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,4096,65536,0.48127999901771545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,3584,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,2560,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,2048,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,12288,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,10240,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,16384,0.12697599828243256
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,8192,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,3584,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,4096,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,5120,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,7168,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,3072,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,12288,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,16384,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,10240,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,5120,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,7168,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,8192,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3584,65536,0.4638719856739044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,4096,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,3072,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,2560,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,3072,65536,0.4607999920845032
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,3584,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,12288,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,8192,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2560,65536,0.3532800078392029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,10240,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,4096,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,5120,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,7168,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,3584,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,3072,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,16384,0.07782399654388428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,12288,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,10240,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,8192,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,16384,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,5120,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,7168,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,3072,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,3584,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,4096,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,2048,65536,0.2877439856529236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,12288,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,8192,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,7168,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1536,65536,0.26419198513031006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,5120,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,10240,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,3584,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,16384,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,4096,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,10240,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,12288,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,16384,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,5120,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,8192,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,7168,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,4096,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,3072,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,3584,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,1024,65536,0.13926400244235992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,10240,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,5120,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,12288,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,16384,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,7168,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,4096,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,256,65536,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,512,65536,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,8192,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,10240,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,5120,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,16384,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,12288,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,160,128,65536,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,4096,0.2969599962234497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,7168,0.4935680031776428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,8192,0.5611519813537598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,12288,0.8243200182914734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,2048,0.15564799308776855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,3584,0.25702399015426636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,512,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,1024,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,5120,0.35942399501800537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,1536,0.12185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,256,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,2560,0.1884160041809082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,128,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,3072,0.22220799326896667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,12288,0.23244799673557281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,8192,0.16281600296497345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,10240,0.19763199985027313
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,4096,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,3584,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,10240,0.694271981716156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,3072,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,7168,0.14847999811172485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,5120,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,16384,0.3020800054073334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,2560,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,1536,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,256,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,2048,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,512,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,1024,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,128,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,7168,0.1157120019197464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,12288,0.17919999361038208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,16384,0.23449599742889404
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,65536,16384,1.0813440084457397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,8192,0.12390399724245071
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,10240,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,16384,65536,1.1141120195388794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,5120,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,3584,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,3072,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,4096,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,2048,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,2560,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,1536,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,256,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,8192,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,5120,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,16384,0.1945600062608719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,7168,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,10240,0.12595200538635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,4096,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,12288,65536,0.8919039964675903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,12288,0.16383999586105347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,2048,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,3584,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,1536,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,2560,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,3072,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,1024,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,10240,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,7168,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,5120,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,16384,0.16076800227165222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,10240,65536,0.694271981716156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,4096,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,8192,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,3584,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,3072,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,12288,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,1536,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,2048,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,2560,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,1024,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,12288,0.11878400295972824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,8192,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,16384,0.16076800227165222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,5120,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,3072,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,7168,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,4096,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,8192,65536,0.6574079990386963
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,3584,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,10240,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,2048,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,2560,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,1024,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,1536,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,12288,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,8192,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,5120,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,10240,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,7168,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,7168,65536,0.5908480286598206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,4096,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,16384,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,3584,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,3072,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,10240,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,16384,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,5120,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,7168,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,8192,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,5120,65536,0.4618239998817444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,12288,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,4096,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,3584,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,3072,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,5120,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,7168,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,8192,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,10240,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,4096,65536,0.4044800102710724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,3584,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,4096,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,16384,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,12288,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,3072,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,12288,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,5120,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,10240,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,16384,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,8192,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,4096,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,3584,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,3072,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,7168,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3584,65536,0.36454400420188904
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,12288,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,10240,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,8192,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,16384,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,4096,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,5120,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,7168,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,3584,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,3072,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,3072,65536,0.24780799448490143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,16384,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2560,65536,0.24371199309825897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,5120,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,12288,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,7168,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,8192,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,4096,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,10240,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,3584,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,3072,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,5120,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,12288,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,10240,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,2048,65536,0.24063999950885773
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,7168,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,8192,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,16384,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,4096,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,2560,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,3072,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,12288,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,7168,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,16384,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,8192,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,5120,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,10240,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,4096,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,3072,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,3584,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1536,65536,0.21401600539684296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,12288,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,7168,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,16384,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,10240,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,8192,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,4096,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,1024,65536,0.13926400244235992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,12288,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,5120,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,10240,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,16384,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,3584,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,512,65536,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,256,65536,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,12288,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,7168,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,16384,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,10240,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,4096,0.2949120104312897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,5120,0.35942399501800537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,12288,0.8202239871025085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,7168,0.4915199875831604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,128,128,65536,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,10240,0.6963199973106384
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,1024,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,3584,0.25600001215934753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,1536,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,3072,0.22323200106620789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,16384,1.0782719850540161
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,2048,0.15564799308776855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,128,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,256,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,512,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,2560,0.18943999707698822
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,65536,8192,0.5591040253639221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,10240,0.19251200556755066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,8192,0.16179199516773224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,5120,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,3584,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,4096,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,2560,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,3072,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,7168,0.1525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,2048,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,1024,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,256,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,512,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,16384,0.29900801181793213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,1536,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,12288,0.2303999960422516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,7168,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,5120,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,10240,0.14847999811172485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,8192,0.131071999669075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,16384,0.23552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,4096,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,3072,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,3584,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,2560,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,2048,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,1536,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,12288,0.17919999361038208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,256,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,12288,0.1443839967250824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,10240,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,16384,65536,1.1079679727554321
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,8192,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,5120,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,7168,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,12288,65536,0.8939520120620728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,16384,0.187391996383667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,4096,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,2560,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,3584,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,3072,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,2048,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,1536,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,1024,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,7168,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,10240,0.10342399775981903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,12288,0.12390399724245071
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,8192,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,16384,0.159743994474411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,4096,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,3584,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,5120,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,3072,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,1536,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,1024,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,2048,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,2560,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,12288,0.11673600226640701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,7168,0.07782399654388428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,16384,0.16179199516773224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,8192,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,10240,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,4096,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,3584,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,5120,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,2560,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,2048,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,3072,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,1536,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,10240,65536,0.6901760101318359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,8192,65536,0.5959680080413818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,7168,65536,0.5785599946975708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,16384,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,5120,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,8192,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,7168,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,12288,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,4096,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,10240,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,3584,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,1536,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,3072,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,7168,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,5120,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,12288,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,10240,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,8192,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,4096,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,3584,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,16384,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,3072,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,5120,65536,0.3676159977912903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,12288,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,16384,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,10240,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,5120,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,4096,65536,0.3553279936313629
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,7168,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,3584,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,8192,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,4096,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,3072,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,12288,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,7168,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,5120,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,4096,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,10240,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,16384,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3584,65536,0.3543039858341217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,8192,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,3584,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,2560,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,128,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,12288,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,16384,0.20377600193023682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,10240,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,5120,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,7168,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,8192,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,4096,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,2560,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,2048,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,12288,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2560,65536,0.24473600089550018
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,16384,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,10240,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,7168,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,5120,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,4096,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,8192,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,2048,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,3072,65536,0.24780799448490143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,2560,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,8192,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,7168,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,10240,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,12288,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,16384,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,4096,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,5120,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,2560,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,2048,65536,0.16793599724769592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,3072,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,128,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,12288,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1536,65536,0.16793599724769592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,16384,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,7168,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,8192,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,10240,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,4096,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,5120,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,3584,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,2048,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,12288,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,5120,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,7168,0.13926400244235992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,16384,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,8192,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,10240,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,3584,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,4096,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,1024,65536,0.1443839967250824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,12288,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,512,65536,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,16384,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,10240,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,12288,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,256,65536,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,8192,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,16384,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,5120,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,10240,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,7168,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,1536,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,2560,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,96,128,65536,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,8192,0.5591040253639221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,4096,0.2887679934501648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,7168,0.4915199875831604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,3072,0.22118400037288666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,10240,0.6952959895133972
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,12288,0.8181759715080261
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,3584,0.25702399015426636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,2560,0.1884160041809082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,16384,1.0751999616622925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,5120,0.3563520014286041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,1536,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,2048,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,256,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,128,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,512,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,65536,1024,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,12288,0.2303999960422516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,8192,0.16179199516773224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,4096,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,16384,0.29900801181793213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,5120,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,10240,0.19251200556755066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,7168,0.1443839967250824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,3072,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,3584,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,2048,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,2560,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,1536,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,1024,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,512,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,12288,0.17612800002098083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,5120,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,7168,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,8192,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,4096,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,3584,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,16384,0.23654399812221527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,3072,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,1536,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,2048,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,10240,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,2560,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,1024,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,16384,65536,1.1089919805526733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,8192,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,5120,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,12288,0.14643199741840363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,10240,0.12595200538635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,7168,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,4096,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,16384,0.18636800348758698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,3584,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,2560,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,3072,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,12288,65536,0.8888319730758667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,1536,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,2048,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,1024,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,12288,0.12390399724245071
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,16384,0.159743994474411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,5120,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,7168,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,10240,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,8192,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,3584,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,4096,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,2560,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,1536,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,2048,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,3072,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,10240,65536,0.6871039867401123
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,8192,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,16384,0.15667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,10240,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,8192,65536,0.5806080102920532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,7168,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,4096,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,12288,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,3584,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,2560,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,3072,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,5120,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,1536,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,2048,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,12288,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,5120,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,7168,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,16384,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,8192,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,4096,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,10240,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,3072,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,3584,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,12288,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,16384,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,10240,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,7168,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,8192,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,5120,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,3072,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,3584,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,2560,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,4096,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,4096,65536,0.3543039858341217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,12288,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,7168,65536,0.5785599946975708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,8192,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,5120,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,7168,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,16384,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,10240,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,3072,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,4096,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,3584,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,5120,65536,0.37273600697517395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3584,65536,0.35123199224472046
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,12288,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,10240,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,16384,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,7168,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,8192,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,5120,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,4096,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,3072,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,2560,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,3584,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,2048,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,7168,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,5120,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,10240,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,8192,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,4096,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,12288,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,16384,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,3072,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,2560,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,3072,65536,0.24780799448490143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,7168,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,12288,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,5120,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,10240,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2560,65536,0.24371199309825897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,8192,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,4096,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,16384,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,3072,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,1536,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,2560,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,12288,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,10240,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,7168,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,8192,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,4096,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,16384,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,5120,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,2048,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,2560,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,2048,65536,0.23756800591945648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,16384,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,10240,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,12288,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,7168,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,8192,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,5120,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,4096,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,3072,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,3584,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,2560,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1536,65536,0.20582400262355804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,1024,65536,0.14028799533843994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,10240,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,16384,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,12288,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,5120,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,3584,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,4096,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,10240,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,8192,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,5120,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,16384,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,512,65536,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,128,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,12288,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,256,65536,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,10240,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,4096,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,16384,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,5120,0.2303999960422516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,4096,0.1812479943037033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,8192,0.3614720106124878
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,10240,0.449535995721817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,7168,0.31334400177001953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,16384,0.6901760101318359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,80,128,65536,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,2048,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,12288,0.5222399830818176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,3072,0.14643199741840363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,1536,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,3584,0.16383999586105347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,256,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,2560,0.12595200538635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,128,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,512,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,65536,1024,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,12288,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,4096,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,8192,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,5120,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,10240,0.131071999669075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,7168,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,16384,0.1955839991569519
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,3584,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,3072,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,1536,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,2560,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,1024,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,2048,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,128,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,512,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,8192,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,7168,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,12288,0.1157120019197464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,5120,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,16384,0.15667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,10240,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,3584,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,4096,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,3072,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,1536,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,2560,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,1024,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,10240,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,8192,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,12288,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,7168,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,16384,0.12390399724245071
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,16384,65536,0.7229440212249756
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,5120,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,4096,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,3584,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,12288,65536,0.5611519813537598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,1536,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,3072,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,2048,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,7168,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,10240,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,5120,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,12288,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,8192,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,16384,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,4096,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,2560,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,1536,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,10240,65536,0.4423680007457733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,10240,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,12288,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,8192,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,16384,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,5120,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,7168,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,4096,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,3072,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,3584,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,2048,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,12288,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,8192,65536,0.40140798687934875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,16384,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,8192,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,7168,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,10240,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,5120,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,3584,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,4096,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,7168,65536,0.3850240111351013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,3072,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,2560,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,12288,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,5120,65536,0.2662400007247925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,7168,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,5120,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,10240,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,8192,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,3072,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,16384,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,4096,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,2560,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,12288,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,10240,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,4096,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,7168,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,5120,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,8192,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,16384,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,3584,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,2560,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,4096,65536,0.25600001215934753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,10240,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,12288,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,8192,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3584,65536,0.24473600089550018
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,5120,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,4096,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,16384,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,7168,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,3072,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,3584,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,12288,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,16384,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,10240,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,8192,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,5120,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,7168,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,4096,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,3584,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,12288,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,7168,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,10240,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,8192,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,3072,65536,0.16793599724769592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,16384,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,4096,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,3584,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,5120,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2560,65536,0.1525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,12288,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,10240,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,7168,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,5120,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,8192,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,4096,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,16384,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,512,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,12288,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,16384,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,8192,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,2048,65536,0.14336000382900238
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,5120,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,7168,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,10240,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,3584,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,4096,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1536,65536,0.1382399946451187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,1024,65536,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,12288,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,8192,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,10240,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,16384,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,12288,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,16384,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,512,65536,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,8192,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,10240,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,256,65536,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,12288,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,5120,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,8192,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,10240,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,7168,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,4096,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,16384,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,64,128,65536,0.11980800330638885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,5120,0.22732800245285034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,4096,0.17919999361038208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,8192,0.4259839951992035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,7168,0.317440003156662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,3072,0.1454080045223236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,2048,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,1536,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,3584,0.1658879965543747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,512,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,10240,0.4423680007457733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,1024,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,2560,0.12083200365304947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,256,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,12288,0.5191680192947388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,16384,0.681984007358551
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,65536,128,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,5120,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,12288,0.158720001578331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,8192,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,10240,0.12800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,4096,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,7168,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,3584,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,3072,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,2560,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,2048,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,1024,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,1536,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,16384,0.19046400487422943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,5120,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,10240,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,8192,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,7168,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,12288,0.11366400122642517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,16384,0.15462400019168854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,4096,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,3584,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,2560,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,1536,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,2048,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,1024,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,3072,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,16384,65536,0.6993920207023621
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,12288,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,5120,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,8192,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,10240,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,7168,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,16384,0.11980800330638885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,3584,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,4096,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,3072,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,2560,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,1536,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,2048,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,12288,65536,0.5283839702606201
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,12288,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,8192,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,10240,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,4096,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,5120,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,7168,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,16384,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,3072,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,2560,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,3584,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,10240,65536,0.4208639860153198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,10240,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,12288,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,16384,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,8192,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,7168,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,8192,65536,0.3819519877433777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,3072,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,2560,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,4096,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,5120,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,3584,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,128,0.21606400609016418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,12288,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,10240,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,7168,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,8192,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,5120,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,16384,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,4096,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,3584,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,2560,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,7168,65536,0.3696640133857727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,3072,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,12288,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,10240,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,5120,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,7168,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,16384,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,8192,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,2560,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,2048,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,4096,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,5120,65536,0.2590720057487488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,12288,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,4096,65536,0.24985599517822266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,8192,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,7168,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,10240,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,5120,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,16384,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,4096,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,3584,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,12288,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,7168,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,10240,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,5120,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,8192,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,3584,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,4096,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,2560,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,16384,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3584,65536,0.24985599517822266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,12288,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,7168,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,10240,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,5120,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,8192,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,16384,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,3072,65536,0.16179199516773224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,3584,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,4096,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,2560,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,12288,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,8192,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2560,65536,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,5120,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,7168,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,10240,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,4096,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,3584,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,2048,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,16384,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,2560,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,12288,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,8192,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,7168,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,5120,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,16384,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,10240,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,3584,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,4096,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,2048,65536,0.14131200313568115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,12288,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,7168,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,5120,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,16384,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,8192,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1536,65536,0.14233599603176117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,12288,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,5120,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,7168,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,8192,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,10240,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,16384,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,1024,65536,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,512,65536,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,12288,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,10240,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,16384,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,256,65536,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,12288,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,5120,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,16384,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,10240,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,48,128,65536,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,4096,0.17817600071430206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,5120,0.2181120067834854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,3584,0.16076800227165222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,8192,0.347135990858078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,7168,0.2979840040206909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,3072,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,2048,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,1536,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,12288,0.49663999676704407
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,512,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,16384,0.6563839912414551
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,2560,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,1024,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,256,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,128,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,65536,10240,0.4188160002231598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,12288,0.14336000382900238
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,7168,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,5120,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,8192,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,10240,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,3584,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,4096,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,16384,0.1884160041809082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,3072,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,2560,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,1024,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,2048,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,1536,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,5120,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,8192,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,10240,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,12288,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,7168,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,16384,0.13414399325847626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,4096,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,3584,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,3072,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,2560,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,2048,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,12288,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,8192,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,10240,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,7168,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,16384,0.11673600226640701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,16384,65536,0.6594560146331787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,5120,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,4096,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,12288,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,8192,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,7168,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,12288,65536,0.47411200404167175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,10240,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,5120,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,3584,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,4096,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,16384,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,3072,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,2560,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,1536,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,10240,65536,0.3962880074977875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,12288,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,8192,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,7168,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,10240,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,4096,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,16384,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,5120,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,3584,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,3072,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,8192,65536,0.3399679958820343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,12288,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,8192,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,10240,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,7168,65536,0.30617600679397583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,5120,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,7168,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,4096,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,16384,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,3072,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,2560,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,12288,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,8192,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,5120,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,16384,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,7168,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,10240,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,5120,65536,0.2242559939622879
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,3584,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,4096,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,2560,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,12288,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,10240,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,5120,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,8192,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,4096,65536,0.19763199985027313
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,7168,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,4096,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,3584,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,16384,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,2560,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,8192,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,10240,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,7168,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,12288,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,5120,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,4096,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3584,65536,0.18227200210094452
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,16384,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,12288,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,7168,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,8192,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,5120,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,3072,65536,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,10240,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,16384,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,4096,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,2048,0.2099200040102005
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,12288,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,10240,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,5120,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,7168,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,16384,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,8192,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,4096,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2560,65536,0.1382399946451187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,12288,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,7168,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,5120,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,8192,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,2048,65536,0.12595200538635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,10240,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,3584,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,4096,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,12288,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,8192,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,5120,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,4096,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,1536,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1536,65536,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,12288,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,1024,65536,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,5120,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,16384,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,8192,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,10240,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,3584,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,512,65536,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,12288,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,10240,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,4096,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,12288,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,16384,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,256,65536,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,7168,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,8192,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,5120,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,32,128,65536,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,4096,0.16486400365829468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,5120,0.2088959962129593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,3584,0.15462400019168854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,7168,0.289792001247406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,8192,0.36454400420188904
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,1536,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,2048,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,1024,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,10240,0.40959998965263367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,16384,0.6676480174064636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,2560,0.1157120019197464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,3072,0.14131200313568115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,128,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,256,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,512,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,65536,12288,0.4925439953804016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,4096,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,5120,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,7168,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,12288,0.131071999669075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,8192,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,16384,0.18227200210094452
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,10240,0.11366400122642517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,3584,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,3072,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,1536,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,1024,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,2560,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,2048,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,12288,0.21299199759960175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,5120,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,16384,0.13926400244235992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,7168,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,8192,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,10240,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,4096,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,2560,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,3584,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,3072,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,1024,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,7168,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,10240,0.07782399654388428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,12288,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,8192,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,4096,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,5120,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,3072,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,16384,65536,0.6502400040626526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,16384,0.11264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,12288,65536,0.4679679870605469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,1024,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,8192,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,12288,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,7168,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,16384,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,5120,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,10240,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,10240,65536,0.41471999883651733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,4096,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,3584,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,2560,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,1536,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,12288,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,7168,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,5120,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,8192,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,16384,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,10240,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,3584,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,4096,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,8192,65536,0.33689600229263306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,2560,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,128,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,8192,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,7168,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,12288,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,10240,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,5120,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,7168,65536,0.3041279911994934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,16384,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,4096,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,3584,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,2048,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,3072,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,12288,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,5120,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,7168,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,10240,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,8192,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,5120,65536,0.21913599967956543
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,16384,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,3072,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,2560,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,3584,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,4096,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,5120,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,8192,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,12288,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,10240,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,7168,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,16384,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,4096,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,3584,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,2048,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,3072,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,4096,65536,0.17715199291706085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,2560,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,12288,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,8192,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,5120,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,16384,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,10240,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,4096,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,3584,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,128,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3584,65536,0.16486400365829468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,12288,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,10240,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,5120,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,16384,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,7168,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,4096,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,3584,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,3072,65536,0.14336000382900238
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2560,65536,0.130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,12288,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,5120,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,16384,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,7168,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,12288,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,10240,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,5120,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,4096,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,8192,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,3584,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,2048,65536,0.12595200538635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,12288,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1536,65536,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,5120,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,7168,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,8192,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,10240,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,16384,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,3584,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,256,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,12288,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,7168,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,8192,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,1024,65536,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,5120,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,16384,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,12288,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,512,65536,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,5120,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,10240,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,16384,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,3584,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,2048,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,12288,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,5120,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,16384,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,256,65536,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,8192,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,10240,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,3584,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,3072,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,512,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,16,128,65536,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,4096,0.16179199516773224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,3584,0.14950400590896606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,5120,0.2027519941329956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,7168,0.28467199206352234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,8192,0.3307519853115082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,10240,0.4106239974498749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,2560,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,1024,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,2048,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,1536,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,3072,0.12902399897575378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,512,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,12288,0.4925439953804016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,256,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,65536,16384,0.6512640118598938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,5120,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,10240,0.11878400295972824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,7168,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,4096,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,8192,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,12288,0.12800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,16384,0.26316800713539124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,3072,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,3584,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,2048,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,2560,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,1536,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,1024,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,256,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,7168,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,5120,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,12288,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,8192,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,10240,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,16384,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,4096,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,3584,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,3072,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,2560,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,2048,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,12288,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,8192,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,10240,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,16384,0.11468800157308578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,7168,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,5120,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,4096,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,16384,65536,0.634880006313324
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,12288,65536,0.477183997631073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,12288,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,8192,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,10240,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,5120,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,7168,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,16384,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,3584,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,4096,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,2560,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,10240,65536,0.3973119854927063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,8192,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,12288,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,7168,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,10240,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,16384,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,5120,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,2560,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,2048,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,3072,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,4096,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,3584,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,8192,65536,0.3328000009059906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,12288,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,7168,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,10240,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,8192,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,16384,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,4096,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,3072,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,5120,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,7168,65536,0.30617600679397583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,512,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,1536,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,12288,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,8192,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,7168,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,10240,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,5120,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,16384,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,4096,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,3584,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,5120,65536,0.2027519941329956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,12288,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,7168,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,10240,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,8192,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,16384,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,5120,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,4096,65536,0.17715199291706085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,3584,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,2048,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,4096,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,2560,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,12288,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,7168,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,8192,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,5120,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,10240,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,16384,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3584,65536,0.16179199516773224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,12288,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,5120,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,10240,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,8192,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,16384,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,3072,65536,0.14847999811172485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,12288,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,8192,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,16384,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,7168,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,10240,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,5120,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2560,65536,0.130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,256,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,12288,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,8192,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,7168,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,5120,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,10240,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,2048,65536,0.1157120019197464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,2048,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,4096,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,12288,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,10240,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,7168,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,16384,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,4096,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,8192,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,2048,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,256,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1536,65536,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,12288,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,16384,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,10240,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,1024,65536,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,5120,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,8192,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,4096,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,3584,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,2560,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,3072,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,2048,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,12288,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,8192,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,5120,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,16384,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,7168,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,10240,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,512,65536,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,3584,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,3072,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,4096,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,1024,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,1536,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,2048,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,12288,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,7168,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,5120,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,16384,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,10240,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,256,65536,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,3072,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,4096,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,2560,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,2048,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,1536,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,3584,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,8,128,65536,0.12185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,4096,0.17203199863433838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,5120,0.21913599967956543
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,7168,0.2979840040206909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,8192,0.3399679958820343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,3072,0.13414399325847626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,3584,0.1443839967250824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,2560,0.11264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,2048,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,10240,0.41676801443099976
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,12288,0.4904960095882416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,256,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,128,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,1024,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,512,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,1536,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,65536,16384,0.6225919723510742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,8192,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,10240,0.1157120019197464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,5120,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,4096,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,7168,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,12288,0.131071999669075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,16384,0.1730560064315796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,3584,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,1024,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,2048,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,3072,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,1536,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,2560,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,5120,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,8192,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,7168,0.10342399775981903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,16384,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,12288,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,10240,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,4096,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,3584,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,2560,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,3072,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,2048,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,8192,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,10240,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,7168,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,12288,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,16384,0.11468800157308578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,4096,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,3584,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,5120,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,2560,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,16384,65536,0.6410239934921265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,3072,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,12288,65536,0.4853760004043579
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,12288,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,10240,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,5120,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,8192,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,7168,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,16384,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,4096,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,3584,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,10240,65536,0.40038400888442993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,12288,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,8192,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,7168,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,10240,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,16384,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,8192,65536,0.3420160114765167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,4096,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,3584,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,5120,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,12288,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,5120,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,7168,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,10240,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,8192,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,16384,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,4096,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,7168,65536,0.30105599761009216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,12288,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,7168,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,10240,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,8192,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,4096,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,16384,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,5120,65536,0.21401600539684296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,5120,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,12288,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,8192,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,10240,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,16384,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,5120,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,7168,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,4096,65536,0.17510400712490082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,12288,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,10240,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,16384,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3584,65536,0.1669120043516159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,3584,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,4096,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,12288,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,8192,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,5120,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,7168,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,16384,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,3584,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,2048,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,3072,65536,0.14233599603176117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,12288,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,5120,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,8192,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,7168,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,10240,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,16384,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,4096,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2560,65536,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,3072,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,2048,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,12288,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,8192,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,5120,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,16384,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,7168,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,10240,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,2560,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,3072,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,2048,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,4096,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,512,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,1024,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,2048,65536,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,12288,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,8192,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,16384,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,7168,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,5120,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1536,65536,0.11264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,10240,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,4096,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,3584,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,3072,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,2560,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,1536,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,2048,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,1024,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,12288,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,7168,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,8192,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,5120,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,16384,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,10240,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,2560,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,2048,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,3072,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,3584,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,4096,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,1024,65536,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,1536,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,1024,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,8192,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,10240,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,5120,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,7168,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,12288,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,512,65536,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,16384,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,3584,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,4096,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,3072,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,2560,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,1536,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,1024,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,12288,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,8192,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,5120,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,7168,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,10240,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,16384,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,4096,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,3584,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,2560,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,3072,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,512,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,1024,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,256,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,2048,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,1536,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,256,65536,0.11673600226640701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,4,128,65536,0.11673600226640701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,4096,0.1730560064315796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,5120,0.20787200331687927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,3584,0.1525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,7168,0.27955201268196106
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,8192,0.32051199674606323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,10240,0.38092800974845886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,2048,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,1536,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,3072,0.13209599256515503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,2560,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,1024,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,12288,0.4567039906978607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,256,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,512,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,65536,16384,0.6379520297050476
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,5120,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,7168,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,4096,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,8192,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,12288,0.12800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,10240,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,2560,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,3584,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,3072,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,2048,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,16384,0.17612800002098083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,1536,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,5120,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,12288,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,8192,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,10240,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,7168,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,16384,0.13209599256515503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,4096,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,3072,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,3584,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,12288,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,8192,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,7168,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,10240,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,16384,0.11366400122642517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,5120,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,4096,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,3584,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,2560,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,16384,65536,0.6246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,3072,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,12288,65536,0.48230400681495667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,12288,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,5120,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,8192,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,10240,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,7168,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,16384,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,3584,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,4096,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,10240,65536,0.3973119854927063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,12288,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,8192,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,7168,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,10240,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,5120,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,4096,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,3584,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,16384,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,2048,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,512,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,8192,65536,0.33484798669815063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,8192,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,12288,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,10240,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,7168,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,16384,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,5120,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,7168,65536,0.2969599962234497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,12288,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,8192,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,5120,65536,0.21401600539684296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,10240,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,7168,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,5120,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,16384,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,8192,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,12288,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,7168,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,10240,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,16384,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,4096,65536,0.17612800002098083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,12288,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,7168,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,8192,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,10240,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,16384,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,3584,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3584,65536,0.16179199516773224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,16384,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,10240,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,2048,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,3072,65536,0.14233599603176117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,12288,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2560,65536,0.12800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,5120,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,10240,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,16384,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,2048,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,128,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,12288,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,16384,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,8192,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,2048,65536,0.1157120019197464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,3072,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,2048,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,1024,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,8192,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1536,65536,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,2048,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,8192,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,12288,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,5120,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,4096,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,1024,65536,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,2560,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,2048,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,1536,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,3072,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,1024,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,12288,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,10240,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,8192,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,16384,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,5120,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,3072,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,4096,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,2048,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,512,65536,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,1536,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,12288,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,8192,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,7168,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,256,65536,0.15667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,10240,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,5120,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,4096,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,16384,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,2560,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,3072,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,1024,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,2048,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,1536,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,2,128,65536,0.15564799308776855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,4096,0.1658879965543747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,5120,0.2088959962129593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,3584,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,3072,0.130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,2560,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,12288,0.45875200629234314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,8192,0.30720001459121704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,7168,0.2764799892902374
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,2048,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,10240,0.37887999415397644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,512,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,1024,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,128,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,256,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,1536,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,65536,16384,0.6256639957427979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,5120,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,4096,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,12288,0.12902399897575378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,10240,0.11264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,8192,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,7168,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,16384,0.17715199291706085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,3584,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,2560,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,3072,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,2048,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,1536,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,7168,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,8192,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,5120,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,16384,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,12288,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,10240,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,3584,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,4096,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,3072,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,2560,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,8192,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,10240,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,16384,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,12288,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,7168,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,16384,65536,0.6410239934921265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,5120,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,4096,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,1536,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,12288,65536,0.4679679870605469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,12288,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,5120,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,8192,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,7168,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,10240,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,16384,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,4096,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,2048,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,3584,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,1024,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,10240,65536,0.3901439905166626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,12288,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,7168,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,8192,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,10240,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,4096,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,16384,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,3584,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,5120,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,8192,65536,0.3102720081806183
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,12288,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,5120,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,7168,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,8192,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,10240,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,16384,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,7168,65536,0.2887679934501648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,12288,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,10240,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,8192,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,5120,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,16384,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,5120,65536,0.21401600539684296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,12288,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,10240,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,8192,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,7168,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,16384,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,4096,65536,0.17817600071430206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,7168,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,12288,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,10240,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3584,65536,0.16076800227165222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,16384,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,10240,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,7168,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,8192,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,16384,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,2048,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,3072,65536,0.14131200313568115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,12288,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,8192,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2560,65536,0.12697599828243256
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,10240,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,7168,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,16384,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,5120,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,3584,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,2048,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,5120,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,12288,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,16384,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,2048,65536,0.11468800157308578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,4096,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,3072,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,2048,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,1024,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,12288,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,8192,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1536,65536,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,3072,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,16384,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,1024,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,2048,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,12288,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,8192,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,16384,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,10240,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,5120,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,1024,65536,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,3072,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,1024,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,1536,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,2048,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,7168,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,12288,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,5120,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,16384,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,8192,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,10240,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,3072,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,3584,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,512,65536,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,2048,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,1536,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,256,65536,0.15667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,12288,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,8192,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,10240,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,16384,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,7168,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,3584,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,4096,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,3072,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,1536,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,2048,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,1024,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int8_wo,1,128,65536,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,4096,17.17043113708496
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,3584,15.10092830657959
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,5120,21.343231201171875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,8192,33.81760025024414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,2560,10.93120002746582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,2048,8.838144302368164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,7168,29.657087326049805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,3072,13.012991905212402
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,128,1.0199040174484253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,512,2.5927679538726807
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,256,1.511423945426941
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,1024,4.684800148010254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,1536,6.7645440101623535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,8192,8.478719711303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,10240,42.28096008300781
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,10240,10.561535835266113
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,16384,16.79769515991211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,4096,4.305920124053955
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,5120,5.389311790466309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,12288,12.63923168182373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,7168,7.436287879943848
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,1536,1.7018879652023315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,2560,2.7637760639190674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,16384,67.04230499267578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,3584,3.79801607131958
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,128,0.26316800713539124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,512,0.6594560146331787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,3072,3.284991979598999
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,1024,1.1816960573196411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,256,0.38604798913002014
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,2048,2.2220799922943115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,8192,6.366208076477051
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,7168,5.5859198570251465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,10240,7.925759792327881
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,4096,3.235840082168579
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,5120,4.019199848175049
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,16384,12.686335563659668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,65536,12288,50.62451171875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,3584,2.8436479568481445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,2048,1.679360032081604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,12288,9.485312461853027
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,2560,2.0602879524230957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,128,0.19968000054359436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,3072,2.4698879718780518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,512,0.49663999676704407
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,256,0.29183998703956604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,1024,0.8898559808731079
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,1536,1.2851200103759766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,7168,4.721663951873779
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,12288,8.0250883102417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,5120,3.3996798992156982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,4096,2.7340800762176514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,8192,5.360640048980713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,3072,2.0725760459899902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,3584,2.401279926300049
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,2560,1.7356799840927124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,2048,1.405951976776123
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,1024,0.749567985534668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,10240,6.676479816436768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,512,0.41574400663375854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,128,0.16793599724769592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,256,0.24473600089550018
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,1536,1.082368016242981
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,65536,42.24204635620117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,10240,5.31763219833374
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,10240,16384,10.6178560256958
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,8192,4.25164794921875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,5120,2.6869759559631348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,3584,1.901568055152893
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,7168,3.7416958808898926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,4096,2.1616640090942383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,16384,8.42240047454834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,2048,1.117184042930603
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,2560,1.385472059249878
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,512,0.33587199449539185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,1024,0.5980160236358643
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,1536,0.857088029384613
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,3072,1.6414719820022583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,128,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,256,0.19968000054359436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,12288,6.3354878425598145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,8192,3.803136110305786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,7168,3.33516788482666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,12288,5.664768218994141
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,16384,65536,66.84467315673828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,5120,2.40230393409729
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,3584,1.7008639574050903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,10240,4.7421441078186035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,16384,7.516160011291504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,4096,1.9333120584487915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,2048,1.0024960041046143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,2560,1.2328959703445435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,1536,0.766975998878479
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,1024,0.5345280170440674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,512,0.2959359884262085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,128,0.12083200365304947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,256,0.17510400712490082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,3072,1.4673919677734375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,10240,3.3525760173797607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,8192,2.692095994949341
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,7168,2.362368106842041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,5120,1.7018879652023315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,16384,5.350399971008301
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,12288,4.011007785797119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,4096,1.3701119422912598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,12288,65536,50.25075149536133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,3584,1.2052479982376099
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,2048,0.7096319794654846
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,3072,1.043455958366394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,512,0.21503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,1536,0.5457919836044312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,2560,0.8755199909210205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,1024,0.3819519877433777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,256,0.12902399897575378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,128,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,8192,2.1391360759735107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,10240,2.673664093017578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,12288,3.1836159229278564
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,16384,4.231167793273926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,7168,1.8769919872283936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,4096,1.0885119438171387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,3584,0.9594879746437073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,5120,1.3598719835281372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,2048,0.5877760052680969
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,1024,0.3112959861755371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,3072,0.8284159898757935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,2560,0.6963199973106384
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,512,0.17203199863433838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,1536,0.43724799156188965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,256,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,128,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,5120,65536,21.06470489501953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,8192,65536,34.14527893066406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,8192,1.913856029510498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,10240,2.3808000087738037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,7168,1.679360032081604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,16384,3.79801607131958
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,12288,2.849792003631592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,4096,0.9748479723930359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,5120,1.2113920450210571
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,3584,0.8581119775772095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,2048,0.5089280009269714
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,7168,65536,29.481983184814453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,2560,0.6236159801483154
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,1536,0.3891200125217438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,1024,0.2754560112953186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,256,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,512,0.15462400019168854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,128,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,3072,0.7454720139503479
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,12288,2.5774080753326416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,7168,1.4847999811172485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,8192,1.6783360242843628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,4096,0.8632320165634155
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,5120,1.070080041885376
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,10240,2.0776960849761963
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,3584,0.7587839961051941
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,16384,3.280895948410034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,2048,0.447488009929657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,3072,0.6563839912414551
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,2560,0.5509120225906372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,256,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,512,0.13414399325847626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,1024,0.24371199309825897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,128,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,1536,0.34508800506591797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,4096,65536,16.82636833190918
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,8192,1.358847975730896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,10240,1.6895999908447266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,5120,0.8601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,16384,2.683903932571411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,12288,2.032639980316162
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,7168,1.1970560550689697
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,4096,0.694271981716156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,2048,0.3624959886074066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,1536,0.27852800488471985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,2560,0.44441598653793335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,3584,0.6103039979934692
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,128,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,1024,0.20377600193023682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,256,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,3072,0.5283839702606201
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,512,0.11264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,8192,1.1499520540237427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,7168,1.0117119550704956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,10240,1.3701119422912598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3584,65536,14.866432189941406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,12288,1.6455680131912231
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,5120,0.7188479900360107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,16384,2.155519962310791
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,4096,0.5765119791030884
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,3584,0.5079039931297302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,2048,0.30003198981285095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,1536,0.2314240038394928
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,2560,0.37171199917793274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,1024,0.16076800227165222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,512,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,3072,0.4413439929485321
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,128,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,256,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,3072,65536,12.659711837768555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,8192,0.8806399703025818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,7168,0.7761920094490051
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,10240,1.0864640474319458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,4096,0.45977601408958435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,5120,0.5642240047454834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,12288,1.2861440181732178
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,16384,1.6885759830474854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,3584,0.4044800102710724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,2048,0.2385919988155365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,1024,0.130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,1536,0.18432000279426575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,2560,0.29286399483680725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,3072,0.3491840064525604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,128,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,512,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,256,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,8192,0.5785599946975708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2560,65536,10.65881633758545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,10240,0.7208960056304932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,12288,0.8540160059928894
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,5120,0.38707199692726135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,16384,1.1151360273361206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,7168,0.5089280009269714
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,4096,0.29900801181793213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,3584,0.26214399933815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,2560,0.19968000054359436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,1024,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,2048,0.158720001578331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,3072,0.22937600314617157
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,1536,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,128,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,256,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,512,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,2048,65536,8.424448013305664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,10240,0.3850240111351013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,8192,0.30822399258613586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,12288,0.447488009929657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,4096,0.16281600296497345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,5120,0.2027519941329956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,7168,0.27238398790359497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,3584,0.14745600521564484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,2560,0.11468800157308578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,16384,0.5898240208625793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,3072,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,512,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,1024,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,256,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,2048,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,128,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,1536,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,8192,0.16998399794101715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,10240,0.2017280012369156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,12288,0.2457599937915802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,16384,0.32153600454330444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,7168,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,5120,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1536,65536,6.458367824554443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,4096,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,1536,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,2048,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,3072,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,2560,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,3584,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,512,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,1024,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,256,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,1024,65536,4.253695964813232
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,8192,0.1157120019197464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,7168,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,10240,0.131071999669075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,12288,0.14745600521564484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,5120,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,4096,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,3584,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,16384,0.2017280012369156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,3072,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,1024,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,2048,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,1536,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,2560,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,512,65536,2.287616014480591
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,7168,14.849023818969727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,8192,16.930816650390625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,256,65536,1.240064024925232
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8192,128,65536,0.7106559872627258
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,4096,8.59545612335205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,3584,7.555071830749512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,1536,3.3914880752563477
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,3072,6.515711784362793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,2048,4.426752090454102
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,512,1.3035520315170288
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,256,0.7608320116996765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,5120,10.726400375366211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,1024,2.348031997680664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,128,0.5120000243186951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,2560,5.471231937408447
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,8192,4.25164794921875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,65536,33.530879974365234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,10240,5.295104026794434
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,5120,2.6869759559631348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,16384,8.457216262817383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,4096,2.1616640090942383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,7168,3.729408025741577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,12288,6.3354878425598145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,1536,0.8581119775772095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,2048,1.1161600351333618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,512,0.3338240087032318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,1024,0.5959680080413818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,128,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,256,0.19865599274635315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,2560,1.3783040046691895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,3584,1.9025919437408447
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,16384,3072,1.6455680131912231
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,10240,21.11180877685547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,8192,3.259392023086548
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,7168,2.849792003631592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,16384,33.54828643798828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,10240,4.042751789093018
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,3072,1.2533760070800781
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,16384,6.402048110961914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,4096,1.652735948562622
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,5120,2.056191921234131
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,2560,1.0547200441360474
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,3584,1.4540799856185913
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,2048,0.8550400137901306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,1024,0.45875200629234314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,128,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,12288,4.838399887084961
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,1536,0.6574079990386963
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,256,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,512,0.2529279887676239
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,10240,3.3525760173797607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,8192,2.6941440105438232
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,4096,1.3701119422912598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,5120,1.7182719707489014
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,7168,2.362368106842041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,3072,1.0414079427719116
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,2048,0.7096319794654846
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,16384,5.327871799468994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,3584,1.2052479982376099
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,65536,12288,25.33683204650879
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,2560,0.8744959831237793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,256,0.12902399897575378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,512,0.21401600539684296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,1024,0.37990400195121765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,128,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,1536,0.5468159914016724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,12288,4.0120320320129395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,7168,1.9834879636764526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,8192,2.140160083770752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,5120,1.3527040481567383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,10240,2.662400007247925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,3072,0.8284159898757935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,4096,1.0885119438171387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,16384,4.228096008300781
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,65536,16.893951416015625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,2048,0.5662720203399658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,3584,0.9584640264511108
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,2560,0.6963199973106384
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,128,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,1024,0.3041279911994934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,256,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,512,0.1730560064315796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,1536,0.4352000057697296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,8192,12288,3.1846399307250977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,8192,1.9128320217132568
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,7168,1.6783360242843628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,4096,0.9748479723930359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,10240,2.381824016571045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,16384,3.79801607131958
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,5120,1.210368037223816
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,3584,0.8581119775772095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,1536,0.4034560024738312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,1024,0.27238398790359497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,2048,0.506879985332489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,3072,0.7423999905586243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,2560,0.6225919723510742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,12288,2.849792003631592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,256,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,512,0.15462400019168854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,128,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,7168,65536,15.036416053771973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,8192,1.3619199991226196
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,12288,2.0203518867492676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,5120,0.8611840009689331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,4096,0.6922240257263184
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,10240,1.6875519752502441
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,7168,1.1919360160827637
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,3072,0.5263360142707825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,2560,0.44441598653793335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,3584,0.6123520135879517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,1536,0.27955201268196106
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,2048,0.3614720106124878
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,1024,0.1945600062608719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,512,0.11264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,128,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,256,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,12288,65536,25.1279354095459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,16384,2.68287992477417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,8192,1.1110399961471558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,10240,65536,21.153791427612305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,7168,0.9799680113792419
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,10240,1.371135950088501
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,3072,0.43827199935913086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,4096,0.5806080102920532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,12288,1.6332800388336182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,5120,0.7178239822387695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,3584,0.5079039931297302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,16384,2.1729280948638916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,2048,0.3041279911994934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,512,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,128,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,256,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,1024,0.159743994474411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,2560,0.3696640133857727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,1536,0.2314240038394928
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,10240,1.2288000583648682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,16384,1.938431978225708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,8192,0.9953280091285706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,7168,0.8775680065155029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,5120,0.6440960168838501
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,4096,0.52019202709198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,3584,0.4556800127029419
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,2048,0.27033600211143494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,3072,0.39321601390838623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,2560,0.3317759931087494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,12288,1.4581760168075562
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,512,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,1024,0.1443839967250824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,256,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,1536,0.2088959962129593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,128,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,8192,0.8806399703025818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,5120,65536,10.656767845153809
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,10240,1.0844160318374634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,16384,1.6926720142364502
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,12288,1.2840960025787354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,5120,0.5785599946975708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,7168,0.7802879810333252
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,4096,0.45875200629234314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,2560,0.2949120104312897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,1536,0.18432000279426575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,3072,0.3491840064525604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,3584,0.4044800102710724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,512,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,1024,0.12800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,2048,0.2385919988155365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,128,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,256,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,4096,65536,8.42137622833252
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,8192,0.6901760101318359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,10240,0.8581119775772095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,7168,0.6062080264091492
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,5120,0.43929600715637207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,12288,1.0332159996032715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,16384,1.3670400381088257
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,4096,0.3553279936313629
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,3072,0.27033600211143494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,2048,0.187391996383667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,3584,0.3123199939727783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,2560,0.22835199534893036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,1024,0.10342399775981903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,1536,0.14643199741840363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,256,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,128,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,512,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,8192,0.5816320180892944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3584,65536,7.5386881828308105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,10240,0.7178239822387695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,16384,1.120255947113037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,12288,0.8529919981956482
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,5120,0.38604798913002014
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,3072,0.23654399812221527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,7168,0.5109760165214539
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,4096,0.29900801181793213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,3584,0.26419198513031006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,2560,0.20070399343967438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,2048,0.16281600296497345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,1536,0.12800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,256,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,512,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,1024,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,128,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,3072,65536,6.459392070770264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,10240,0.5611519813537598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,8192,0.45875200629234314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,16384,0.8837119936943054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,4096,0.24063999950885773
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,7168,0.4055039882659912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,5120,0.2979840040206909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,3584,0.21299199759960175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,12288,0.6553599834442139
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,3072,0.18534399569034576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,1536,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,2048,0.130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,2560,0.15667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,512,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,1024,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,128,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,256,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,8192,0.30720001459121704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,16384,0.5857279896736145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,10240,0.3758080005645752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2560,65536,5.3442559242248535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,12288,0.4485119879245758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,4096,0.16281600296497345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,3072,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,5120,0.2017280012369156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,7168,0.27238398790359497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,3584,0.1443839967250824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,2048,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,1024,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,1536,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,512,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,2560,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,256,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,128,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,8192,0.16896000504493713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,12288,0.24063999950885773
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,10240,0.19763199985027313
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,16384,0.317440003156662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,3584,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,5120,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,4096,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,7168,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,2048,65536,4.250624179840088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,1536,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,2048,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,1024,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,2560,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,3072,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,256,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,512,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1024,65536,2.2558720111846924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,8192,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,7168,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,10240,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,5120,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,12288,0.13209599256515503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,16384,0.17203199863433838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,1536,65536,3.3648641109466553
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,3584,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,4096,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,3072,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,2560,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,2048,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,1536,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,512,65536,1.1499520540237427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,7168,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,8192,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,10240,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,12288,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,3584,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,4096,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,5120,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,3072,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,16384,0.11264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,2048,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,2560,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,1536,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,256,65536,0.6379520297050476
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4096,128,65536,0.3983359932899475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,4096,4.311039924621582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,5120,5.3534722328186035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,8192,8.516608238220215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,3584,3.791872024536133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,7168,7.437312126159668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,1024,1.1806720495224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,2048,2.228224039077759
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,2560,2.746367931365967
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,512,0.6553599834442139
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,128,0.2611199915409088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,256,0.38707199692726135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,3072,3.2665600776672363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,10240,2.6644480228424072
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,1536,1.7018879652023315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,10240,10.555392265319824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,5120,1.4202879667282104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,12288,3.2030720710754395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,4096,1.0895359516143799
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,8192,2.1391360759735107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,16384,4.226047992706299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,7168,1.8759679794311523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,3072,0.8284159898757935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,2048,0.5662720203399658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,16384,16.914432525634766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,3584,0.9625599980354309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,1024,0.3041279911994934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,512,0.17203199863433838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,256,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,128,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,2560,0.6983680129051208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,1536,0.4362240135669708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,7168,1.4796799421310425
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,8192,1.679360032081604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,5120,1.0690560340881348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,4096,0.8632320165634155
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,10240,2.081792116165161
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,65536,12288,12.637184143066406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,3584,0.9338880181312561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,3072,0.6563839912414551
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,16384,3.26963210105896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,2048,0.449535995721817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,2560,0.5539839863777161
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,512,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,1024,0.24166400730609894
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,12288,2.4821760654449463
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,128,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,1536,0.3461120128631592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,256,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,10240,1.696768045425415
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,12288,2.0346879959106445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,4096,0.6983680129051208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,5120,0.8642560243606567
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,8192,1.358847975730896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,7168,1.1919360160827637
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,3584,0.6154239773750305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,2560,0.44646400213241577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,1024,0.1955839991569519
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,1536,0.2815999984741211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,2048,0.3624959886074066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,256,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,128,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,512,0.11264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,3072,0.5314559936523438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,10240,1.4315520524978638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,16384,2.683903932571411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,8192,1.1141120195388794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,5120,0.7208960056304932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,16384,2.164736032485962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,7168,0.9789440035820007
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,4096,0.5836799740791321
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,3584,0.5109760165214539
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,2560,0.37273600697517395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,65536,8.491007804870605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,12288,1.632256031036377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,2048,0.30003198981285095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,1024,0.1658879965543747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,512,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,256,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,128,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,3072,0.4403199851512909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,8192,1536,0.2314240038394928
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,10240,1.2288000583648682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,16384,2.027519941329956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,8192,0.9983999729156494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,16384,65536,16.7587833404541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,5120,0.6604800224304199
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,3584,0.45875200629234314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,3072,0.3962880074977875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,7168,0.8785920143127441
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,2048,0.27238398790359497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,2560,0.3328000009059906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,4096,0.52019202709198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,1024,0.1454080045223236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,512,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,256,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,128,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,1536,0.20787200331687927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,12288,1.0291199684143066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,12288,1.462272047996521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,16384,1.3629440069198608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,7168,65536,7.570432186126709
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,4096,0.3553279936313629
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,10240,0.8581119775772095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,8192,0.6901760101318359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,7168,0.6062080264091492
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,5120,0.4413439929485321
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,3584,0.4065279960632324
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,3072,0.27238398790359497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,2560,0.2303999960422516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,1024,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,512,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,2048,0.1884160041809082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,256,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,128,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,1536,0.14745600521564484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,12288,65536,12.65766429901123
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,10240,0.7178239822387695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,8192,0.5806080102920532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,7168,0.532480001449585
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,12288,0.8540160059928894
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,16384,1.1223039627075195
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,5120,0.3850240111351013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,4096,0.30003198981285095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,1536,0.12390399724245071
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,2048,0.16383999586105347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,3072,0.22937600314617157
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,3584,0.2693119943141937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,2560,0.1945600062608719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,512,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,256,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,128,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,1024,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,10240,65536,10.61580753326416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,8192,0.5283839702606201
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,5120,65536,5.342207908630371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,10240,0.6481919884681702
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,4096,0.2764799892902374
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,12288,0.7649279832839966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,16384,0.9983999729156494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,5120,0.3409920036792755
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,3584,0.2467840015888214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,7168,0.4638719856739044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,2048,0.14847999811172485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,3072,0.21401600539684296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,2560,0.18227200210094452
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,512,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,1536,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,1024,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,256,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,128,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,10240,0.5601279735565186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,12288,0.6778879761695862
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,4096,65536,4.255743980407715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,7168,0.4055039882659912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,4096,0.24268800020217896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,3584,0.21401600539684296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,8192,0.45875200629234314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,3072,0.18534399569034576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,2048,0.12902399897575378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,2560,0.1658879965543747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,1536,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,5120,0.2979840040206909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,1024,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,16384,0.881663978099823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,256,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,512,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,128,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,10240,0.44441598653793335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,16384,0.6963199973106384
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,8192,0.3563520014286041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,7168,0.31539198756217957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,5120,0.22937600314617157
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,4096,0.18534399569034576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,3584,0.16998399794101715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,3072,0.1443839967250824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,12288,0.5273600220680237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,2048,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,1536,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,512,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,2560,0.12390399724245071
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,1024,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,128,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,256,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,12288,0.46489599347114563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,10240,0.3768320083618164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,16384,0.6144000291824341
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,8192,0.30720001459121704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,5120,0.2017280012369156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,4096,0.1669120043516159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,7168,0.27033600211143494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,3072,0.131071999669075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,2560,0.11366400122642517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,3584,0.1443839967250824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3584,65536,3.8195199966430664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,2048,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,1024,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,256,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,512,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,1536,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,128,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,3072,65536,3.363840103149414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,10240,0.2887679934501648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,8192,0.24268800020217896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,5120,0.15667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,7168,0.21401600539684296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2048,65536,2.254848003387451
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,4096,0.12800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,12288,0.3461120128631592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,3584,0.11366400122642517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,16384,0.46592000126838684
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,1024,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,3072,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,2048,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,2560,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,512,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,128,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,1536,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,256,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,7168,0.14950400590896606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,10240,0.20582400262355804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,8192,0.16896000504493713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,2560,65536,2.698240041732788
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,12288,0.2396160066127777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,5120,0.11366400122642517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,4096,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,3584,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,2560,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,3072,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,2048,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,1536,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,1024,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,256,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,16384,0.31539198756217957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,512,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,7168,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,8192,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,10240,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,12288,0.130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,5120,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,4096,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,3584,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,3072,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,2560,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,1024,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,16384,0.16896000504493713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,1536,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,2048,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,512,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1024,65536,1.1499520540237427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,8192,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,12288,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,10240,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,7168,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,1536,65536,1.7920000553131104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,4096,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,16384,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,3584,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,5120,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,2560,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,3072,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,2048,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,8192,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,7168,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,10240,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,512,65536,0.6215680241584778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,5120,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,12288,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,4096,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,16384,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,2560,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,256,65536,0.3911679983139038
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,8192,4.25267219543457
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,4096,2.164736032485962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,3584,1.901568055152893
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,3072,1.6414719820022583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2048,128,65536,0.24268800020217896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,2048,1.1182080507278442
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,2560,1.3803520202636719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,1024,0.596992015838623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,512,0.3317759931087494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,1536,0.8581119775772095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,128,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,256,0.19763199985027313
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,5120,2.6900479793548584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,16384,2.257920026779175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,8192,1.1489280462265015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,7168,0.9799680113792419
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,5120,0.7188479900360107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,7168,3.733504056930542
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,4096,0.5806080102920532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,3584,0.5120000243186951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,2560,0.37068799138069153
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,3072,0.4423680007457733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,1536,0.23449599742889404
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,10240,1.3752319812774658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,512,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,1024,0.16486400365829468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,2048,0.29900801181793213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,128,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,256,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,12288,1.6353280544281006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,12288,1.2861440181732178
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,10240,5.315584182739258
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,8192,0.8785920143127441
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,10240,1.0864640474319458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,7168,0.7782400250434875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,16384,8.458239555358887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,16384,1.6844799518585205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,4096,0.45875200629234314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,5120,0.5642240047454834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,2560,0.2949120104312897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,3584,0.4065279960632324
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,3072,0.3543039858341217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,1024,0.12800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,2048,0.2385919988155365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,256,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,128,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,512,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,1536,0.18432000279426575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,8192,0.6901760101318359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,12288,1.0240000486373901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,10240,0.8611840009689331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,4096,0.35737600922584534
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,5120,0.43929600715637207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,7168,0.6082559823989868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,65536,12288,6.3354878425598145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,3584,0.31436800956726074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,3072,0.27238398790359497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,2048,0.187391996383667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,1536,0.14745600521564484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,2560,0.2303999960422516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,1024,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,16384,1.363968014717102
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,512,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,256,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,128,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,12288,0.851967990398407
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,8192,0.6062080264091492
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,10240,0.7393280267715454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,16384,1.117184042930603
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,7168,0.5314559936523438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,4096,0.3112959861755371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,10240,65536,5.342207908630371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,3584,0.26419198513031006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,2560,0.19968000054359436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,5120,0.3686400055885315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,3072,0.22630399465560913
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,1536,0.12595200538635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,2048,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,256,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,1024,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,512,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,128,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,12288,0.7823359966278076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,10240,0.6563839912414551
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,16384,1.0301439762115479
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,8192,0.5253120064735413
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,5120,0.347135990858078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,4096,0.2836480140686035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,8192,65536,4.46668815612793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,3584,0.2457599937915802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,7168,0.46489599347114563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,3072,0.21401600539684296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,2560,0.1812479943037033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,1024,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,512,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,2048,0.14745600521564484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,256,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,1536,0.11673600226640701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,128,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,16384,65536,8.419327735900879
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,12288,0.5273600220680237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,10240,0.47308799624443054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,8192,0.35839998722076416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,7168,0.3256320059299469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,16384,0.7434239983558655
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,7168,65536,3.8062078952789307
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,4096,0.1955839991569519
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,2048,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,5120,0.22937600314617157
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,1536,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,2560,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,3072,0.14336000382900238
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,3584,0.1658879965543747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,512,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,1024,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,256,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,128,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,12288,65536,6.459392070770264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,12288,0.44646400213241577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,10240,0.3758080005645752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,8192,0.30617600679397583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,16384,0.6144000291824341
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,7168,0.27033600211143494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,4096,0.16793599724769592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,2560,0.11264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,3072,0.13209599256515503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,1536,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,3584,0.1443839967250824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,5120,0.19968000054359436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,512,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,1024,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,2048,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,256,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,128,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,5120,65536,2.7207679748535156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,7168,0.24371199309825897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,10240,0.33587199449539185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,5120,0.18227200210094452
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,8192,0.2744320034980774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,4096,0.14950400590896606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,3584,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,4096,65536,2.25382399559021
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,1024,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,2560,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,12288,0.3983359932899475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,2048,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,3072,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,16384,0.5263360142707825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,1536,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,256,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,512,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,128,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,5120,0.16793599724769592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,8192,0.24268800020217896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,10240,0.28569599986076355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,16384,0.5089280009269714
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,4096,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,3584,0.12185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,7168,0.21299199759960175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,3072,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,2560,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,1536,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,2048,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,1024,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,512,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,256,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,128,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,12288,0.3461120128631592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,7168,0.17612800002098083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,8192,0.19763199985027313
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,10240,0.24063999950885773
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,5120,0.1382399946451187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,12288,0.2826240062713623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,16384,0.3686400055885315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3072,65536,1.8206720352172852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,4096,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,1024,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,2048,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,3584,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,2560,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,1536,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,3072,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,512,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,256,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,12288,0.24371199309825897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,7168,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,8192,0.16998399794101715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2560,65536,1.3752319812774658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,5120,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,10240,0.20377600193023682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,3584,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,4096,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,3584,65536,2.0203518867492676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,2560,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,3072,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,16384,0.3112959861755371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,2048,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,1536,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,1024,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,512,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,5120,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,8192,0.13926400244235992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,7168,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,10240,0.15462400019168854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,12288,0.18329599499702454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,3584,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,4096,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,16384,0.2519040107727051
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,2048,65536,1.1479040384292603
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,1536,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,3072,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,1024,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,2048,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,2560,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,512,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,128,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,12288,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,7168,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,10240,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,5120,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,8192,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,16384,0.16793599724769592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,4096,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,3584,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,3072,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,2048,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,2560,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,1536,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,8192,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,7168,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,12288,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,10240,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,16384,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,4096,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,5120,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,3072,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,3584,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,2560,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,2048,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,1536,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,12288,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,10240,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,8192,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1536,65536,0.9830399751663208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,16384,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,1024,65536,0.6154239773750305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,3072,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,4096,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,7168,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,5120,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,2048,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,1536,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,2560,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,512,65536,0.3911679983139038
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,12288,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,8192,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,5120,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,7168,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,16384,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,10240,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,3584,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,4096,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,3072,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,256,65536,0.23552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,10240,4.064256191253662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1024,128,65536,0.16179199516773224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,3072,1.2584960460662842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,4096,1.6599040031433105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,2048,0.857088029384613
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,3584,1.45305597782135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,5120,2.052095890045166
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,2560,1.0557440519332886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,1024,0.45875200629234314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,512,0.25088000297546387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,8192,3.2481279373168945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,128,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,256,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,1536,0.6584320068359375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,12288,1.30457603931427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,10240,1.0885119438171387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,7168,2.8487679958343506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,7168,0.7802879810333252
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,4096,0.46694400906562805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,8192,0.881663978099823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,3584,0.4126720130443573
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,3072,0.3491840064525604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,2048,0.2457599937915802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,1024,0.13414399325847626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,2560,0.2959359884262085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,5120,0.5713919997215271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,16384,1.7059839963912964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,512,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,128,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,1536,0.18329599499702454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,256,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,16384,65536,6.464511871337891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,12288,0.9451519846916199
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,8192,0.6543359756469727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,16384,1.2830719947814941
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,7168,0.5765119791030884
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,3072,0.26009601354599
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,2560,0.22015999257564545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,4096,0.3409920036792755
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,1536,0.1382399946451187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,3584,0.29900801181793213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,5120,0.4331519901752472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,512,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,1024,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,2048,0.17817600071430206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,256,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,128,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,10240,0.8048639893531799
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,12288,0.8581119775772095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,12288,65536,4.9100799560546875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,16384,1.1376639604568481
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,7168,0.5099520087242126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,5120,0.3676159977912903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,4096,0.2979840040206909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,3072,0.22835199534893036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,2560,0.19353599846363068
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,16384,6.427648067474365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,8192,0.5775359869003296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,10240,0.7167999744415283
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,1536,0.1157120019197464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,2048,0.158720001578331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,512,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,1024,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,256,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,128,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,3584,0.26419198513031006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,12288,0.6737920045852661
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,7168,0.42905598878860474
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,10240,65536,4.0447998046875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,4096,0.2519040107727051
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,3584,0.22220799326896667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,3072,0.19353599846363068
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,5120,0.29900801181793213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,8192,0.46694400906562805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,2560,0.16383999586105347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,2048,0.130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,1024,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,512,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,256,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,128,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,1536,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,10240,0.5785599946975708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,65536,12288,4.8363518714904785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,16384,0.881663978099823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,16384,0.7700480222702026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,10240,0.502784013748169
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,5120,0.26009601354599
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,8192,65536,3.580928087234497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,4096,0.22118400037288666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,3072,0.1658879965543747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,7168,0.35839998722076416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,3584,0.19046400487422943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,2560,0.14745600521564484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,8192,0.4065279960632324
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,1536,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,2048,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,1024,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,512,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,256,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,128,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,12288,0.5857279896736145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,12288,0.47206398844718933
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,7168,0.31334400177001953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,8192,0.3102720081806183
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,16384,0.6973440051078796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,5120,0.22732800245285034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,4096,0.18534399569034576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,2048,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,10240,0.4413439929485321
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,3072,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,2560,0.12185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,1536,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,3584,0.16383999586105347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,7168,65536,2.929663896560669
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,1024,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,128,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,512,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,256,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,5120,0.16281600296497345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,8192,0.2457599937915802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,7168,0.21606400609016418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,10240,0.289792001247406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,16384,0.4925439953804016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,3584,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,4096,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,3072,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,2560,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,1536,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,5120,65536,2.0336639881134033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,1024,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,256,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,512,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,2048,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,128,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,12288,0.3563520014286041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,5120,0.14336000382900238
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,10240,0.2590720057487488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,4096,0.11878400295972824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,3584,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,7168,0.19046400487422943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,8192,0.21401600539684296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,3072,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,4096,65536,1.810431957244873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,2560,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,1024,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,512,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,2048,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,256,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,128,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,1536,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,12288,0.3092480003833771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,16384,0.4044800102710724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,7168,0.17715199291706085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,5120,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,12288,0.25600001215934753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,8192,0.24780799448490143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,16384,0.38809600472450256
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,10240,0.28467199206352234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,3584,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,3072,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,2048,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,4096,0.12697599828243256
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,2560,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,1536,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,1024,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,256,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,512,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,7168,0.14745600521564484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,12288,0.23654399812221527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,8192,0.1669120043516159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,4096,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,10240,0.22937600314617157
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,5120,0.130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,3584,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3072,65536,1.3844480514526367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,16384,0.3092480003833771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,3072,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,1024,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,2048,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,2560,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,512,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,1536,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,128,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,8192,0.1443839967250824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,7168,0.12185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,10240,0.16998399794101715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,16384,0.25804799795150757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,3584,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,4096,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,12288,0.1945600062608719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,5120,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,3072,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,2048,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,2560,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,1536,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,1024,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,3584,65536,1.551360011100769
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,512,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,12288,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,8192,0.13414399325847626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2048,65536,0.9553920030593872
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,7168,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,10240,0.12595200538635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,5120,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,16384,0.24883200228214264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,2048,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,2560,0.2385919988155365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,3584,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,4096,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,3072,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,1024,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,1536,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,2560,65536,1.1479040384292603
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,8192,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,7168,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,10240,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,16384,0.13619199395179749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1536,65536,0.9390079975128174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,12288,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,5120,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,3072,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,3584,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,4096,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,2560,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,1536,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,12288,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,5120,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,8192,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,7168,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,10240,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,16384,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,3584,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,4096,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,128,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,1024,65536,0.5939199924468994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,7168,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,8192,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,10240,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,12288,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,16384,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,5120,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,4096,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,512,65536,0.2754560112953186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,3584,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,7168,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,5120,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,8192,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,16384,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,10240,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,4096,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,256,65536,0.16383999586105347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,3584,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,5120,1.358847975730896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,10240,2.6808319091796875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,3584,0.9605119824409485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,16384,4.245503902435303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,2048,0.5683199763298035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,4096,1.090559959411621
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,768,128,65536,0.11264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,1024,0.3031040132045746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,2560,0.6983680129051208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,512,0.17100800573825836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,256,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,1536,0.43724799156188965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,128,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,3072,0.8325120210647583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,12288,0.8785920143127441
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,10240,0.7362560033798218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,8192,2.1370880603790283
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,16384,1.1509759426116943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,8192,0.6082559823989868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,7168,0.5386239886283875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,5120,0.3840000033378601
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,4096,0.3184640109539032
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,2560,0.1945600062608719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,2048,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,1536,0.12390399724245071
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,7168,1.8851840496063232
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,256,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,128,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,1024,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,512,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,3072,0.2303999960422516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,3584,0.26316800713539124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,12288,0.6553599834442139
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,10240,0.5591040253639221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,8192,0.4607999920845032
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,4096,0.24371199309825897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,5120,0.29900801181793213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,3584,0.21503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,16384,65536,4.25164794921875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,3072,0.18636800348758698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,2048,0.130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,7168,0.4055039882659912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,512,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,2560,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,1024,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,1536,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,256,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,128,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,16384,0.8847360014915466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,12288,0.562175989151001
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,8192,0.37068799138069153
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,12288,65536,3.359744071960449
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,10240,0.4710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,16384,0.6983680129051208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,7168,0.33689600229263306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,4096,0.1955839991569519
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,2560,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,3584,0.16896000504493713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,3072,0.14745600521564484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,2048,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,512,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,1024,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,128,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,256,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,1536,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,5120,0.22937600314617157
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,12288,0.4485119879245758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,8192,0.3184640109539032
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,10240,0.3829759955406189
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,16384,0.6133760213851929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,65536,12288,3.1877119541168213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,4096,0.16793599724769592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,5120,0.20684799551963806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,7168,0.27033600211143494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,3072,0.12595200538635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,10240,65536,2.712575912475586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,1536,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,512,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,2048,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,256,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,3584,0.14336000382900238
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,1024,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,2560,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,128,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,12288,0.3973119854927063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,8192,0.2744320034980774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,7168,0.2519040107727051
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,10240,0.33587199449539185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,16384,0.5283839702606201
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,4096,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,5120,0.1812479943037033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,3584,0.13414399325847626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,8192,65536,2.254848003387451
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,3072,0.11878400295972824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,2560,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,1536,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,256,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,512,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,1024,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,2048,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,128,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,12288,0.2908160090446472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,8192,0.19865599274635315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,5120,0.130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,7168,0.17612800002098083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,10240,0.24268800020217896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,16384,0.3676159977912903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,4096,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,3584,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,2560,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,2048,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,3072,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,1536,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,1024,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,512,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,7168,65536,2.0336639881134033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,12288,0.24166400730609894
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,5120,0.11673600226640701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,10240,0.19763199985027313
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,8192,0.16793599724769592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,16384,0.3164159953594208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,7168,0.14950400590896606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,4096,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,2560,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,3584,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,1024,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,1536,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,2048,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,3072,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,512,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,12288,0.21503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,5120,65536,1.371135950088501
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,8192,0.1525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,7168,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,16384,0.28672000765800476
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,5120,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,4096,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,3584,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,2560,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,2048,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,1024,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,10240,0.19353599846363068
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,3072,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,1536,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,512,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,4096,65536,1.227776050567627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,5120,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,12288,0.2088959962129593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,7168,0.12595200538635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,8192,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,10240,0.1669120043516159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,16384,0.25702399015426636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,4096,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,3584,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,2048,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,3072,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,2560,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,1536,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,256,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3584,65536,1.0885119438171387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,10240,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,8192,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,5120,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,7168,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,12288,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,4096,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,16384,0.20377600193023682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,3072,65536,0.9390079975128174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,2048,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,3584,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,2560,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,3072,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,1536,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,8192,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,10240,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,5120,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,7168,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,4096,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,3584,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,3072,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,12288,0.12902399897575378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,2560,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,16384,0.17203199863433838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,2048,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,1536,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,16384,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,7168,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,8192,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,10240,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,12288,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2048,65536,0.6184960007667542
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,2560,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,4096,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,5120,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,3072,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,3584,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,1536,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,1024,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,2048,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,2560,65536,0.754688024520874
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,12288,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,10240,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,5120,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,16384,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,7168,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,8192,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,4096,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,3584,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,3072,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,2560,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,2048,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,12288,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,8192,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,7168,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,16384,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,10240,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,5120,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,4096,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1536,65536,0.49459201097488403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,3584,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,1536,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,1024,65536,0.3829759955406189
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,12288,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,5120,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,10240,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,8192,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,7168,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,16384,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,4096,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,3584,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,3072,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,10240,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,256,65536,0.16383999586105347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,3072,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,3584,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,512,65536,0.23552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,4096,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,4096,0.8622080087661743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,5120,1.0711040496826172
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,8192,1.6936960220336914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,3072,0.6574079990386963
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,3584,0.7598080039024353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,16384,3.3413119316101074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,2048,0.4505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,1024,0.24063999950885773
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,512,0.13619199395179749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,128,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,512,128,65536,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,256,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,12288,0.6717439889907837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,16384,0.8837119936943054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,2560,0.5539839863777161
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,1536,0.34508800506591797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,7168,0.43007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,5120,0.3102720081806183
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,10240,0.5785599946975708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,8192,0.48742398619651794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,3584,0.22118400037288666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,3072,0.18534399569034576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,2560,0.16281600296497345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,4096,0.24166400730609894
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,2048,0.12902399897575378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,1024,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,256,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,512,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,128,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,1536,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,16384,65536,3.3658881187438965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,7168,1.4827519655227661
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,16384,0.6574079990386963
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,5120,0.22835199534893036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,8192,0.35225600004196167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,4096,0.18636800348758698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,7168,0.3112959861755371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,10240,0.4352000057697296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,10240,2.1043200492858887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,2048,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,3584,0.16486400365829468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,3072,0.1443839967250824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,12288,0.48230400681495667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,2560,0.12185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,1536,0.07782399654388428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,512,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,256,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,128,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,65536,2.4780800342559814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,12288,1024,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,12288,0.5355520248413086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,5120,0.19968000054359436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,10240,0.3973119854927063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,7168,0.2805759906768799
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,16384,0.6952959895133972
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,8192,0.3563520014286041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,3072,0.14336000382900238
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,2048,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,3584,0.14336000382900238
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,4096,0.18636800348758698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,2560,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,1024,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,128,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,512,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,256,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,1536,0.07782399654388428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,10240,0.28569599986076355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,7168,0.22732800245285034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,8192,0.25600001215934753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,16384,0.49561598896980286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,65536,12288,2.5128960609436035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,5120,0.16281600296497345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,10240,65536,2.0357120037078857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,3072,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,4096,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,2560,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,1536,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,3584,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,1024,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,2048,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,256,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,128,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,512,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,12288,0.3543039858341217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,12288,0.30617600679397583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,10240,0.2590720057487488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,8192,0.22937600314617157
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,5120,0.14233599603176117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,7168,0.19046400487422943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,4096,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,3584,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,2560,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,1536,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,1024,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,2048,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,8192,65536,1.810431957244873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,3072,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,512,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,256,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,128,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,16384,0.40755200386047363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,5120,0.12595200538635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,7168,0.14847999811172485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,12288,0.24268800020217896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,8192,0.16793599724769592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,10240,0.20582400262355804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,3072,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,2560,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,2048,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,3584,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,4096,0.10342399775981903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,1024,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,1536,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,512,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,7168,65536,1.59334397315979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,16384,0.36556801199913025
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,12288,0.1955839991569519
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,5120,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,7168,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,10240,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,4096,0.07782399654388428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,16384,0.2549760043621063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,8192,0.14131200313568115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,2560,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,3072,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,3584,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,2048,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,1536,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,512,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,1024,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,5120,65536,1.1509759426116943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,12288,0.16793599724769592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,5120,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,7168,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,4096,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,8192,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,10240,0.14028799533843994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,3584,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,4096,65536,0.9871360063552856
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,3072,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,2560,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,16384,0.2252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,1024,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,1536,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,2048,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,5120,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,7168,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3584,65536,0.8499199748039246
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,8192,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,16384,0.20582400262355804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,10240,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,12288,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,3072,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,3584,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,4096,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,2560,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,2048,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,1536,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,1024,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,512,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,12288,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,7168,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,8192,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,10240,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,5120,0.07782399654388428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,3584,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,4096,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,2560,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,3072,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,16384,0.20479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,3072,65536,0.7157760262489319
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,2048,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,1536,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,1024,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,12288,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,5120,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,7168,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,8192,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,10240,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,4096,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,3584,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,3072,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,1536,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,2048,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,16384,0.13619199395179749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,8192,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2048,65536,0.4976640045642853
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,7168,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,10240,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,12288,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,5120,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,3072,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,4096,0.2314240038394928
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,2560,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,16384,0.11366400122642517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,3584,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,2048,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,1536,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,2560,65536,0.6307839751243591
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,12288,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,8192,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,10240,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,5120,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1536,65536,0.4864000082015991
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,16384,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,7168,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,3072,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,4096,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,3584,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,12288,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,8192,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,10240,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,7168,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,16384,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,4096,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,3584,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,1024,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,1024,65536,0.30822399258613586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,12288,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,8192,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,10240,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,7168,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,4096,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,512,65536,0.16486400365829468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,8192,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,5120,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,256,65536,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,7168,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,10240,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,4096,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,384,128,65536,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,7168,0.9820160269737244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,12288,1.6332800388336182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,10240,1.3803520202636719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,2560,0.3696640133857727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,3072,0.43929600715637207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,16384,2.1575679779052734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,2048,0.30003198981285095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,1536,0.22937600314617157
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,256,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,512,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,1024,0.16179199516773224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,4096,0.5806080102920532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,128,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,12288,0.4638719856739044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,8192,0.31334400177001953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,16384,0.596992015838623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,5120,0.7167999744415283
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,3584,0.5089280009269714
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,10240,0.3768320083618164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,65536,8192,1.1141120195388794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,5120,0.20479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,3072,0.13209599256515503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,4096,0.16998399794101715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,2560,0.11264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,3584,0.1443839967250824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,2048,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,1536,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,512,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,1024,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,256,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,128,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,7168,0.27136000990867615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,7168,0.21503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,16384,0.4710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,8192,0.24371199309825897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,16384,65536,2.264064073562622
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,5120,0.15667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,4096,0.12800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,3584,0.11468800157308578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,10240,0.2877439856529236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,3072,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,2560,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,2048,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,12288,0.34508800506591797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,512,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,128,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,1024,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,256,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,1536,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,8192,0.19763199985027313
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,12288,0.28672000765800476
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,7168,0.17612800002098083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,5120,0.130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,10240,0.24473600089550018
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,16384,0.3901439905166626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,4096,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,3584,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,2048,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,3072,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,2560,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,1536,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,1024,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,512,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,256,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,12288,0.23654399812221527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,7168,0.14950400590896606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,8192,0.16896000504493713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,10240,0.19865599274635315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,16384,0.3102720081806183
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,5120,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,4096,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,3584,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,2560,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,3072,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,2048,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,1536,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,512,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,1024,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,12288,0.21503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,10240,0.18432000279426575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,8192,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,16384,0.29286399483680725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,5120,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,4096,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,8192,65536,1.2267520427703857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,3584,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,7168,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,3072,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,10240,65536,1.3731839656829834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,2560,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,12288,65536,1.7848320007324219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,2048,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,1536,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,512,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,1024,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,8192,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,7168,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,12288,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,5120,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,4096,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,3584,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,3072,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,10240,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,2560,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,16384,0.19865599274635315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,7168,65536,1.1079679727554321
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,2048,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,1536,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,1024,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,12288,0.17510400712490082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,8192,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,7168,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,5120,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,16384,0.1669120043516159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,10240,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,4096,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,2048,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,3584,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,1536,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,2560,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,3072,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,12288,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,7168,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,8192,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,10240,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,5120,65536,0.7352319955825806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,4096,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,2560,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,3584,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,16384,0.1658879965543747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,3072,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,5120,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,2048,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,1536,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,1024,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,12288,0.11468800157308578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,4096,65536,0.6164479851722717
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,7168,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,8192,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,16384,0.14131200313568115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,5120,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,3072,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,3584,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,4096,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,10240,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,10240,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,8192,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,16384,0.11366400122642517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3072,65536,0.4864000082015991
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,12288,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,7168,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,5120,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,4096,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,3584,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,2560,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,3072,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,3584,65536,0.6133760213851929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2560,65536,0.39526399970054626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,5120,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,12288,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,8192,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,10240,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,7168,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,4096,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,16384,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,3584,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,2560,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,3072,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,12288,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,5120,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,7168,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,8192,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,10240,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,4096,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,3072,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,2048,65536,0.39526399970054626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,3584,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,16384,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,12288,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,5120,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,8192,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,7168,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,10240,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1536,65536,0.3112959861755371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,16384,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,3072,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,3584,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,4096,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,2560,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,8192,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,7168,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,5120,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,10240,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,16384,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,12288,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,4096,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,3584,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,3072,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,2560,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,1024,65536,0.27238398790359497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,16384,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,10240,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,512,65536,0.16179199516773224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,4096,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,12288,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,5120,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,10240,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,8192,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,16384,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,256,65536,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,4096,0.4607999920845032
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,256,128,65536,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,8192,1.1079679727554321
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,3072,0.3481599986553192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,2560,0.29388800263404846
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,2048,0.23756800591945648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,1536,0.18329599499702454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,1024,0.12800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,256,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,512,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,128,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,5120,0.5683199763298035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,3584,0.4065279960632324
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,12288,0.3758080005645752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,10240,0.29286399483680725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,16384,0.5939199924468994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,7168,0.9758719801902771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,8192,0.24473600089550018
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,7168,0.3112959861755371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,5120,0.158720001578331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,4096,0.32870399951934814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,3584,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,3072,0.10342399775981903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,2048,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,2560,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,1536,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,1024,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,10240,1.369088053703308
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,512,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,256,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,128,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,12288,0.3420160114765167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,8192,0.24473600089550018
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,10240,0.28569599986076355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,16384,2.155519962310791
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,4096,0.12697599828243256
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,5120,0.15667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,3584,0.11468800157308578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,2560,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,2048,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,16384,0.47308799624443054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,7168,0.1658879965543747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,3072,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,65536,12288,1.6302080154418945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,1536,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,1024,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,512,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,12288,0.22015999257564545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,7168,0.14847999811172485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,8192,0.16896000504493713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,10240,0.18943999707698822
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,16384,0.3266560137271881
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,5120,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,4096,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,3072,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,3584,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,2560,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,2048,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,1536,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,512,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,1024,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,12288,0.2017280012369156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,10240,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,8192,0.1658879965543747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,16384,0.30720001459121704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,5120,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,7168,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,3584,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,4096,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,3072,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,2048,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,1536,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,2560,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,512,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,1024,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,16384,65536,2.25382399559021
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,12288,0.1740799993276596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,10240,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,16384,0.22630399465560913
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,12288,65536,1.324031949043274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,8192,0.12185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,5120,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,7168,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,4096,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,3584,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,3072,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,2048,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,2560,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,1536,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,1024,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,256,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,7168,65536,0.869376003742218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,10240,0.11878400295972824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,8192,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,16384,0.1669120043516159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,12288,0.130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,10240,65536,1.3701119422912598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,7168,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,5120,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,3584,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,2048,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,4096,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,3072,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,1536,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,8192,65536,1.1479040384292603
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,2560,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,8192,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,5120,65536,0.6328319907188416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,16384,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,5120,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,10240,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,7168,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,4096,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,3584,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,12288,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,3072,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,2560,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,7168,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,10240,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,16384,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,5120,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,8192,0.07782399654388428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,4096,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,4096,65536,0.5150719881057739
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,2560,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,3584,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,12288,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,2048,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,3072,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,12288,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,8192,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,10240,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,16384,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,5120,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,4096,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,7168,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,3584,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,3072,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3584,65536,0.4976640045642853
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,12288,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,8192,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,5120,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,7168,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,3072,65536,0.48844799399375916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,16384,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,10240,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,4096,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,3584,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,2560,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,3072,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,12288,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,7168,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,10240,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,8192,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,5120,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,4096,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2560,65536,0.392192006111145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,16384,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,2560,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,3584,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,12288,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,5120,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,10240,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,8192,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,7168,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,16384,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,4096,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,3584,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,3072,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,2560,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,2048,65536,0.2754560112953186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,12288,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,8192,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,16384,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,7168,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,5120,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,10240,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,4096,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,3072,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,2560,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,3584,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,12288,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,8192,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,10240,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,7168,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,16384,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,3584,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,4096,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1024,65536,0.1669120043516159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,2560,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,1536,65536,0.31334400177001953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,512,65536,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,10240,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,16384,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,8192,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,3584,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,7168,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,5120,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,5120,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,12288,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,256,65536,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,8192,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,16384,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,10240,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,8192,1.112064003944397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,5120,0.5713919997215271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,4096,0.45875200629234314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,192,128,65536,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,7168,0.9748479723930359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,10240,1.368064045906067
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,12288,1.6291840076446533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,2048,0.23756800591945648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,2560,0.3031040132045746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,16384,2.1575679779052734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,1536,0.18227200210094452
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,1024,0.12800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,256,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,128,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,512,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,10240,0.29286399483680725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,8192,0.25600001215934753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,12288,0.37273600697517395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,7168,0.22732800245285034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,16384,0.4915199875831604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,3584,0.4034560024738312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,4096,0.14131200313568115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,65536,3072,0.35020801424980164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,5120,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,3584,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,2048,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,3072,0.10342399775981903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,2560,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,1024,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,512,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,1536,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,256,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,128,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,5120,0.15667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,10240,0.28467199206352234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,7168,0.1658879965543747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,3584,0.11366400122642517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,16384,0.3461120128631592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,8192,0.24166400730609894
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,4096,0.12800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,3072,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,16384,65536,2.2568960189819336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,2560,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,1536,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,1024,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,512,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,256,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,2048,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,12288,0.34303998947143555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,7168,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,12288,0.22323200106620789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,16384,0.3246079981327057
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,10240,0.18534399569034576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,8192,0.19251200556755066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,5120,0.12595200538635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,4096,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,3584,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,2560,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,2048,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,1536,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,3072,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,1024,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,256,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,512,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,12288,0.1955839991569519
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,7168,0.12083200365304947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,16384,0.30720001459121704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,10240,0.1525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,8192,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,5120,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,3584,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,10240,65536,1.371135950088501
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,3072,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,4096,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,2560,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,2048,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,1536,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,1024,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,12288,0.1669120043516159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,16384,0.22937600314617157
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,7168,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,8192,0.12185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,10240,0.1443839967250824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,5120,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,8192,65536,0.9881600141525269
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,4096,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,3072,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,3584,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,2048,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,1536,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,2560,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,1024,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,512,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,12288,0.12902399897575378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,10240,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,5120,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,12288,65536,1.3158400058746338
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,7168,65536,0.8704000115394592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,8192,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,16384,0.1658879965543747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,7168,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,4096,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,3584,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,2560,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,1536,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,3072,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,12288,0.11264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,8192,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,7168,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,5120,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,5120,65536,0.6307839751243591
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,10240,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,16384,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,3072,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,3584,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,4096,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,2560,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,2048,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,12288,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,10240,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,5120,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,16384,0.13619199395179749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,8192,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,4096,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,7168,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,4096,65536,0.5109760165214539
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,2560,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,3072,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,3584,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,2048,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,12288,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3584,65536,0.4986880123615265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,16384,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,8192,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,10240,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,5120,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,7168,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,3584,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,4096,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,3072,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,2048,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,12288,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,5120,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,7168,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,8192,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,3584,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,3072,65536,0.48844799399375916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,16384,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,4096,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,3072,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,10240,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,1024,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,12288,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,16384,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,10240,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,4096,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2560,65536,0.3911679983139038
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,8192,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,5120,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,7168,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,3584,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,3072,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,12288,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,7168,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,5120,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,10240,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,8192,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,16384,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,4096,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,2560,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,2048,65536,0.2385919988155365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,3584,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,7168,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,12288,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,8192,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1536,65536,0.23756800591945648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,16384,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,10240,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,4096,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,3584,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,2560,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,12288,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,8192,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,7168,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,16384,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,3584,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,4096,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,10240,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,2048,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,1024,65536,0.1658879965543747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,12288,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,8192,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,512,65536,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,16384,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,5120,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,10240,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,8192,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,12288,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,256,65536,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,16384,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,160,128,65536,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,5120,0.3891200125217438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,8192,0.5816320180892944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,4096,0.30105599761009216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,10240,0.7208960056304932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,12288,0.857088029384613
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,3584,0.26521599292755127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,3072,0.22835199534893036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,2560,0.19251200556755066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,1536,0.12390399724245071
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,512,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,128,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,256,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,1024,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,16384,1.117184042930603
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,2048,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,12288,0.24371199309825897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,65536,7168,0.5089280009269714
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,5120,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,8192,0.16896000504493713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,16384,0.317440003156662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,7168,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,4096,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,10240,0.19865599274635315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,3584,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,2048,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,1024,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,3072,0.07782399654388428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,2560,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,1536,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,512,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,128,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,5120,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,12288,0.19660800695419312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,7168,0.11878400295972824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,8192,0.13414399325847626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,10240,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,4096,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,3072,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,2560,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,16384,65536,1.2339199781417847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,2048,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,3584,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,1536,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,16384,0.2519040107727051
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,1024,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,12288,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,5120,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,8192,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,7168,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,16384,0.19865599274635315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,10240,0.131071999669075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,12288,65536,0.9359359741210938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,3072,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,3584,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,2048,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,2560,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,1536,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,4096,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,12288,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,5120,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,7168,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,10240,0.11468800157308578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,16384,0.1669120043516159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,8192,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,3072,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,2048,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,3584,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,1536,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,2560,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,4096,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,1024,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,10240,65536,0.737280011177063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,12288,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,8192,0.11878400295972824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,10240,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,7168,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,16384,0.1658879965543747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,5120,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,4096,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,8192,65536,0.6307839751243591
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,3072,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,2560,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,3584,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,1024,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,2048,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,1536,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,12288,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,8192,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,7168,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,5120,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,10240,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,16384,0.11468800157308578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,4096,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,3584,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,3072,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,7168,65536,0.6297600269317627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,2048,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,12288,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,10240,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,8192,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,5120,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,7168,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,5120,65536,0.3901439905166626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,16384,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,3584,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,3072,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,4096,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,7168,0.13619199395179749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,12288,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,5120,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,8192,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,16384,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,10240,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,4096,65536,0.3901439905166626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,3584,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,4096,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,12288,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,5120,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,7168,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,16384,0.131071999669075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,10240,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,8192,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,4096,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,3584,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,1024,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,12288,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,8192,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,7168,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,10240,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,4096,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,16384,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,5120,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,3584,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3584,65536,0.37990400195121765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,3072,65536,0.4034560024738312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,1024,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2560,65536,0.2662400007247925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,12288,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,4096,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,8192,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,10240,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,5120,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,16384,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,7168,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,3072,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,5120,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,7168,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,8192,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,10240,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,2048,65536,0.27136000990867615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,16384,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,4096,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,12288,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,3584,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,12288,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,7168,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,10240,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,8192,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,5120,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,4096,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1536,65536,0.23654399812221527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,16384,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,3584,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,12288,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,5120,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,8192,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,1024,65536,0.2088959962129593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,10240,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,7168,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,4096,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,16384,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,3584,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,12288,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,5120,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,7168,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,16384,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,512,65536,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,10240,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,256,65536,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,12288,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,16384,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,10240,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,3072,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,256,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,128,128,65536,0.130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,7168,0.5079039931297302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,12288,0.8478720188140869
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,3584,0.26419198513031006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,4096,0.2969599962234497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,2560,0.1945600062608719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,3072,0.22835199534893036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,2048,0.1658879965543747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,1536,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,512,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,5120,0.3696640133857727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,1024,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,256,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,128,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,8192,0.7086079716682434
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,12288,0.24268800020217896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,10240,0.19968000054359436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,8192,0.16793599724769592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,7168,0.14950400590896606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,16384,0.31539198756217957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,5120,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,3584,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,4096,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,3072,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,10240,0.7413759827613831
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,2048,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,2560,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,1024,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,512,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,1536,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,65536,16384,1.1182080507278442
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,12288,0.18534399569034576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,8192,0.30105599761009216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,10240,0.158720001578331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,5120,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,7168,0.11878400295972824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,3584,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,4096,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,2560,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,3072,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,2048,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,1536,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,16384,0.2529279887676239
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,16384,65536,1.2257280349731445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,12288,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,8192,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,10240,0.131071999669075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,5120,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,7168,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,16384,0.19865599274635315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,3072,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,4096,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,3584,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,1536,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,2560,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,2048,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,12288,65536,0.9349120259284973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,12288,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,7168,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,5120,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,10240,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,10240,65536,0.7342079877853394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,8192,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,4096,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,16384,0.1658879965543747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,3584,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,2048,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,2560,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,3072,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,1536,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,1024,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,5120,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,12288,0.12185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,8192,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,10240,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,7168,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,4096,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,3584,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,2560,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,3072,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,16384,0.1658879965543747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,1536,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,2048,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,1024,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,8192,65536,0.6297600269317627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,12288,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,10240,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,8192,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,7168,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,5120,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,4096,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,3584,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,16384,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,3072,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,7168,65536,0.6307839751243591
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,2560,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,2048,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,12288,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,5120,65536,0.41471999883651733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,10240,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,8192,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,7168,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,5120,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,16384,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,3584,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,4096,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,1024,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,12288,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,10240,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,7168,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,8192,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,5120,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,16384,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,4096,65536,0.39526399970054626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,3584,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,4096,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,12288,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,8192,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,7168,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,5120,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,10240,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,4096,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3584,65536,0.3092480003833771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,3584,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,16384,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,2048,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,12288,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,7168,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,8192,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,10240,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,5120,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,3584,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,16384,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,3072,65536,0.30720001459121704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,4096,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,12288,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,5120,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,7168,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2560,65536,0.2662400007247925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,10240,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,16384,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,8192,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,4096,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,3584,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,12288,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,7168,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,16384,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,10240,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,8192,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,4096,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,3584,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,2560,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,2048,65536,0.27033600211143494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,12288,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,5120,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,10240,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,7168,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,8192,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1536,65536,0.16383999586105347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,4096,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,12288,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,16384,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,5120,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,1024,65536,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,3072,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,256,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,512,65536,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,12288,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,16384,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,10240,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,2560,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,2048,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,12288,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,16384,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,10240,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,7168,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,256,65536,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,256,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,4096,0.29900801181793213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,96,128,65536,0.11980800330638885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,8192,0.5775359869003296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,5120,0.3686400055885315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,10240,0.7290880084037781
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,3584,0.26521599292755127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,7168,0.5058559775352478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,12288,0.8458240032196045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,16384,1.1048959493637085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,2560,0.19251200556755066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,3072,0.22732800245285034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,1536,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,512,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,128,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,256,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,1024,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,65536,2048,0.158720001578331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,12288,0.24268800020217896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,8192,0.1669120043516159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,5120,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,7168,0.14847999811172485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,16384,0.31539198756217957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,10240,0.19865599274635315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,3072,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,4096,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,3584,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,2560,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,512,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,1536,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,2048,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,1024,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,12288,0.18329599499702454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,16384,0.24985599517822266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,7168,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,10240,0.1525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,8192,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,5120,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,3584,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,4096,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,2560,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,2048,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,3072,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,16384,65536,1.2288000583648682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,1536,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,12288,0.15667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,16384,0.2017280012369156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,5120,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,8192,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,7168,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,10240,0.130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,4096,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,12288,65536,0.9379839897155762
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,3584,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,2048,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,1536,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,3072,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,2560,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,12288,0.12800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,7168,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,5120,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,16384,0.1669120043516159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,8192,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,10240,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,4096,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,3584,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,3072,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,2560,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,1536,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,2048,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,10240,65536,0.7516160011291504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,12288,0.12185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,8192,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,7168,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,10240,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,4096,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,16384,0.16486400365829468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,5120,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,3584,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,8192,65536,0.6287360191345215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,2048,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,3072,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,1536,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,2560,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,5120,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,7168,65536,0.6297600269317627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,8192,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,7168,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,10240,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,16384,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,12288,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,3584,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,3072,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,4096,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,2560,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,128,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,12288,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,10240,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,5120,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,8192,0.2611199915409088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,7168,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,3584,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,16384,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,5120,65536,0.40857601165771484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,4096,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,12288,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,16384,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,8192,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,10240,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,4096,65536,0.39526399970054626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,5120,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,7168,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,3584,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,4096,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,12288,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,10240,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,7168,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,16384,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,8192,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,4096,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,5120,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,3584,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,3072,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3584,65536,0.30617600679397583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,5120,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,16384,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,7168,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,3072,65536,0.2887679934501648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,12288,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,10240,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,8192,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,4096,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,3584,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,12288,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,10240,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,8192,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,5120,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,7168,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,3584,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,4096,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,16384,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2560,65536,0.26521599292755127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,12288,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,5120,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,7168,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,8192,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,2048,65536,0.27033600211143494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,10240,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,16384,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,4096,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,3584,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,128,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,12288,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,10240,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,5120,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,7168,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,4096,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,3584,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,128,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,12288,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,16384,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1024,65536,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,10240,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,1536,65536,0.16486400365829468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,8192,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,12288,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,16384,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,512,65536,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,3584,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,256,65536,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,12288,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,5120,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,10240,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,16384,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,4096,0.16998399794101715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,7168,0.27750399708747864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,10240,0.38092800974845886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,8192,0.3112959861755371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,12288,0.4505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,5120,0.19968000054359436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,80,128,65536,0.130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,2560,0.12185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,16384,0.5898240208625793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,3072,0.12595200538635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,2048,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,3584,0.1454080045223236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,1024,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,512,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,256,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,128,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,65536,1536,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,7168,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,5120,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,4096,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,12288,0.14336000382900238
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,8192,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,16384,0.16998399794101715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,10240,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,3584,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,3072,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,2560,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,2048,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,1536,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,5120,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,10240,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,12288,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,7168,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,16384,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,8192,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,3584,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,3072,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,4096,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,16384,65536,0.6297600269317627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,5120,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,8192,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,7168,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,10240,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,12288,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,16384,0.12595200538635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,4096,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,3584,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,3072,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,2560,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,1024,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,12288,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,7168,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,10240,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,16384,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,8192,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,4096,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,10240,65536,0.5283839702606201
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,3072,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,5120,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,3584,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,12288,65536,0.5017600059509277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,2560,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,2048,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,1536,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,12288,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,5120,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,16384,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,8192,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,7168,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,4096,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,10240,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,3072,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,3584,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,2560,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,8192,65536,0.5294079780578613
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,10240,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,12288,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,5120,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,7168,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,16384,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,8192,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,4096,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,3072,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,7168,65536,0.3891200125217438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,12288,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,16384,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,8192,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,5120,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,7168,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,10240,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,3584,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,5120,65536,0.27750399708747864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,2560,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,4096,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,3072,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,12288,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,4096,65536,0.2744320034980774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,16384,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,5120,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,8192,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,7168,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,4096,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,10240,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,3072,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,3584,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,2560,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,5120,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,12288,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,7168,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,10240,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3584,65536,0.23654399812221527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,8192,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,4096,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,16384,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,3584,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,12288,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,7168,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,16384,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,4096,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,8192,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,10240,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,3584,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,3072,65536,0.16793599724769592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,2560,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,12288,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,16384,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,8192,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,10240,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,5120,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,7168,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,4096,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,3584,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,3072,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2560,65536,0.16486400365829468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,12288,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,16384,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,10240,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,2048,65536,0.1658879965543747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,5120,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,7168,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,8192,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,4096,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,10240,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,16384,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,8192,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1536,65536,0.16179199516773224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,4096,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,10240,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,8192,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,1024,65536,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,16384,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,12288,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,10240,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,16384,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,512,65536,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,3584,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,2048,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,256,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,12288,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,5120,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,16384,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,256,65536,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,64,128,65536,0.11980800330638885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,5120,0.21094399690628052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,7168,0.2734079957008362
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,3584,0.1443839967250824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,4096,0.1658879965543747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,8192,0.3112959861755371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,12288,0.449535995721817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,512,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,2560,0.12185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,1024,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,1536,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,2048,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,256,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,128,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,16384,0.5857279896736145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,3072,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,12288,0.14131200313568115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,8192,0.14233599603176117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,10240,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,65536,10240,0.37785598635673523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,7168,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,4096,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,16384,0.17203199863433838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,5120,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,3584,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,1536,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,3072,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,2560,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,2048,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,12288,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,5120,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,10240,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,8192,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,16384,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,7168,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,3584,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,16384,65536,0.6195200085639954
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,3072,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,4096,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,7168,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,10240,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,12288,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,8192,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,5120,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,16384,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,4096,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,12288,65536,0.4997119903564453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,3584,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,3072,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,2560,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,2048,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,12288,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,8192,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,7168,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,10240,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,5120,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,16384,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,4096,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,3584,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,3072,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,1536,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,2560,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,2048,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,10240,65536,0.3901439905166626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,7168,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,5120,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,12288,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,8192,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,10240,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,4096,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,16384,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,2560,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,3584,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,3072,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,1536,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,8192,65536,0.3901439905166626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,5120,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,8192,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,7168,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,12288,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,16384,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,7168,65536,0.39423999190330505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,10240,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,2048,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,4096,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,3072,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,3584,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,2560,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,12288,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,10240,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,16384,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,5120,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,7168,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,8192,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,3584,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,4096,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,5120,65536,0.2744320034980774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,10240,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,7168,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,12288,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,4096,65536,0.27033600211143494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,5120,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,8192,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,3072,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,4096,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,2560,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,16384,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,3584,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,8192,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,12288,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,7168,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,10240,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,5120,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3584,65536,0.23449599742889404
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,4096,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,16384,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,2560,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,3584,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,12288,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,5120,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,8192,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,10240,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,7168,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,4096,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,3072,65536,0.1658879965543747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,16384,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,3584,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,12288,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,7168,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,10240,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,16384,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,5120,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,8192,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,4096,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2560,65536,0.16486400365829468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,2048,65536,0.16793599724769592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,12288,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,7168,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,8192,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,5120,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,4096,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,10240,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,3584,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,16384,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,5120,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,16384,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,8192,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,7168,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1536,65536,0.16179199516773224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,1024,65536,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,12288,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,7168,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,8192,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,5120,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,16384,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,12288,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,10240,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,16384,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,7168,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,512,65536,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,12288,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,10240,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,256,65536,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,7168,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,16384,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,3584,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,48,128,65536,0.12083200365304947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,5120,0.13209599256515503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,4096,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,8192,0.20787200331687927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,10240,0.24985599517822266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,12288,0.29286399483680725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,7168,0.17817600071430206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,3584,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,2048,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,3072,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,2560,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,1536,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,256,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,1024,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,512,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,128,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,65536,16384,0.4485119879245758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,12288,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,8192,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,5120,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,10240,0.10342399775981903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,7168,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,4096,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,3584,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,16384,0.15462400019168854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,2560,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,2048,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,3072,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,1536,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,12288,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,7168,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,10240,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,5120,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,8192,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,16384,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,16384,65536,0.3993600010871887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,3584,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,3072,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,4096,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,2560,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,7168,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,10240,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,12288,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,8192,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,5120,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,16384,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,3584,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,12288,65536,0.317440003156662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,4096,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,3072,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,2048,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,12288,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,8192,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,5120,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,7168,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,16384,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,10240,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,2560,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,3072,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,3584,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,4096,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,10240,65536,0.2467840015888214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,1024,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,12288,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,5120,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,10240,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,8192,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,7168,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,16384,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,3072,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,2560,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,3584,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,4096,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,8192,65536,0.23654399812221527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,12288,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,5120,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,7168,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,8192,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,4096,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,10240,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,16384,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,3584,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,3072,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,7168,65536,0.23449599742889404
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,12288,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,7168,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,5120,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,8192,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,16384,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,10240,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,4096,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,3584,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,3072,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,2048,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,5120,65536,0.16486400365829468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,128,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,4096,65536,0.1669120043516159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,12288,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,10240,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,5120,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,4096,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,8192,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,7168,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,16384,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,3584,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,12288,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,16384,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,5120,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,4096,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,8192,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,7168,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,3584,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3584,65536,0.16896000504493713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,8192,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,12288,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,7168,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,16384,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,10240,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,3072,65536,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,4096,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,3072,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,3584,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,12288,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,8192,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,7168,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,16384,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,10240,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,5120,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,4096,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,3584,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2560,65536,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,1536,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,2048,65536,0.13619199395179749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,16384,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,12288,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,5120,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,8192,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,4096,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,12288,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,7168,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,16384,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,4096,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,8192,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,10240,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,3584,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,3072,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1536,65536,0.11366400122642517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,256,0.19353599846363068
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,16384,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,1024,65536,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,12288,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,7168,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,8192,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,5120,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,12288,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,10240,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,5120,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,512,65536,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,16384,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,12288,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,10240,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,256,65536,0.14336000382900238
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,16384,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,5120,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,32,128,65536,0.11878400295972824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,4096,0.10342399775981903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,8192,0.18227200210094452
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,5120,0.13209599256515503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,12288,0.2693119943141937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,3584,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,7168,0.15564799308776855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,2560,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,10240,0.22732800245285034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,3072,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,16384,0.35020801424980164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,256,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,1024,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,1536,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,2048,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,512,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,65536,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,12288,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,4096,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,8192,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,10240,0.07782399654388428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,16384,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,5120,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,7168,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,3584,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,1536,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,2048,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,2560,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,3072,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,10240,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,12288,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,5120,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,7168,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,8192,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,16384,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,4096,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,3072,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,3584,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,2560,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,16384,65536,0.4065279960632324
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,12288,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,8192,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,10240,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,5120,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,7168,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,16384,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,3584,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,4096,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,12288,65536,0.2549760043621063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,1536,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,2048,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,2560,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,10240,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,8192,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,7168,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,12288,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,10240,65536,0.2314240038394928
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,5120,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,16384,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,4096,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,3584,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,12288,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,7168,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,5120,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,16384,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,8192,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,10240,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,3584,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,2560,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,4096,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,8192,65536,0.2099200040102005
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,12288,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,10240,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,8192,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,7168,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,16384,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,3584,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,4096,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,3072,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,7168,65536,0.21094399690628052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,5120,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,7168,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,16384,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,4096,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,10240,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,3584,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,5120,65536,0.1443839967250824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,8192,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,5120,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,10240,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,12288,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,4096,65536,0.1382399946451187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,7168,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,16384,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,4096,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,3584,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,12288,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,8192,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,7168,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,5120,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,10240,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,16384,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3584,65536,0.130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,4096,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,512,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,16384,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,8192,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,5120,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,10240,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,3072,65536,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,3584,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,4096,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,3072,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,512,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,12288,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,8192,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,5120,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,10240,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,4096,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,16384,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,3584,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,3072,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2560,65536,0.11980800330638885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,12288,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,8192,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,16384,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,5120,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,2048,65536,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,1536,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,12288,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,7168,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,16384,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,5120,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,10240,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1536,65536,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,1024,65536,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,12288,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,5120,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,8192,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,16384,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,7168,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,10240,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,12288,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,16384,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,10240,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,512,65536,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,12288,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,5120,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,8192,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,16384,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,7168,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,3072,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,2048,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,256,65536,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,16,128,65536,0.13209599256515503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,4096,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,5120,0.11264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,7168,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,3584,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,8192,0.17715199291706085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,2560,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,2048,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,3072,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,1536,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,10240,0.22118400037288666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,12288,0.2590720057487488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,1024,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,512,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,16384,0.3481599986553192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,65536,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,8192,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,4096,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,5120,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,12288,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,7168,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,10240,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,3584,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,16384,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,3072,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,1536,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,10240,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,12288,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,7168,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,8192,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,5120,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,16384,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,4096,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,3584,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,1536,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,2560,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,16384,65536,0.3481599986553192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,12288,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,8192,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,7168,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,10240,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,5120,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,4096,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,16384,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,12288,65536,0.2539519965648651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,8192,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,12288,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,5120,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,7168,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,10240,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,16384,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,4096,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,2560,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,3072,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,3584,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,10240,65536,0.2252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,12288,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,8192,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,10240,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,7168,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,8192,65536,0.18943999707698822
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,16384,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,4096,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,3584,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,2048,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,2560,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,10240,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,8192,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,12288,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,7168,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,5120,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,4096,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,3584,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,3072,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,7168,65536,0.1884160041809082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,10240,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,8192,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,16384,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,5120,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,7168,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,5120,65536,0.14233599603176117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,3584,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,4096,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,8192,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,12288,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,10240,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,5120,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,7168,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,3584,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,4096,65536,0.130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,12288,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,10240,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,5120,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,7168,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3584,65536,0.12902399897575378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,10240,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,12288,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,8192,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,5120,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,4096,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,3072,65536,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,16384,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,2048,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,12288,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,16384,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,8192,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2560,65536,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,12288,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,8192,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,7168,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,10240,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,16384,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,2048,65536,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,12288,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,7168,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,10240,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1536,65536,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,16384,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,2048,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,2560,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,256,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,1024,65536,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,12288,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,16384,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,5120,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,8192,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,10240,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,2048,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,12288,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,512,65536,0.12185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,7168,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,8192,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,16384,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,10240,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,3584,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,12288,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,7168,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,10240,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,256,65536,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,16384,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,3584,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,2560,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,8,128,65536,0.12185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,4096,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,5120,0.11468800157308578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,3584,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,8192,0.17817600071430206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,7168,0.16281600296497345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,3072,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,2560,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,12288,0.25804799795150757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,10240,0.22015999257564545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,16384,0.3409920036792755
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,2048,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,1024,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,512,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,65536,1536,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,12288,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,5120,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,8192,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,4096,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,7168,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,16384,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,10240,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,3584,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,3072,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,2560,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,2048,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,12288,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,5120,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,8192,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,7168,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,10240,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,16384,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,4096,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,3584,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,2560,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,3072,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,2048,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,12288,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,16384,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,8192,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,16384,65536,0.3543039858341217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,10240,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,7168,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,4096,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,5120,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,3584,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,12288,65536,0.25804799795150757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,2560,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,8192,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,10240,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,7168,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,16384,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,5120,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,12288,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,10240,65536,0.3553279936313629
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,3584,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,4096,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,3072,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,1536,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,8192,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,5120,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,7168,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,12288,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,10240,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,4096,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,3072,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,3584,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,16384,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,8192,65536,0.21606400609016418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,12288,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,5120,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,7168,65536,0.187391996383667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,8192,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,10240,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,7168,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,16384,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,5120,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,7168,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,8192,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,10240,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,1024,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,5120,65536,0.13926400244235992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,12288,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,8192,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,10240,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,5120,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,7168,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,16384,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,4096,65536,0.131071999669075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,5120,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,8192,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,2560,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,4096,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,2048,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3584,65536,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,3072,65536,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,7168,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,8192,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,5120,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,4096,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,16384,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,2560,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,3584,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,2048,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,1024,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,12288,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,8192,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,2048,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,1024,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2560,65536,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,512,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,16384,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,5120,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,7168,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,2048,65536,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,10240,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,8192,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,4096,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,3072,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,2048,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,1536,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,10240,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,12288,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,5120,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,16384,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,7168,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1536,65536,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,8192,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,4096,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,3072,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,2560,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,1536,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,2048,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,1024,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,12288,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,8192,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,16384,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,7168,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,10240,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,5120,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,1024,65536,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,3584,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,4096,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,3072,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,2560,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,512,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,1024,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,1536,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,2048,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,12288,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,5120,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,16384,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,512,65536,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,10240,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,7168,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,8192,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,4096,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,3584,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,3072,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,2560,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,2048,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,1536,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,1024,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,12288,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,7168,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,5120,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,256,65536,0.12185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,8192,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,16384,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,4096,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,10240,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,3584,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,3072,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,2560,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,4,128,65536,0.12185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,4096,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,5120,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,8192,0.17203199863433838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,3584,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,7168,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,10240,0.20582400262355804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,12288,0.2303999960422516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,3072,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,2048,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,1536,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,16384,0.32767999172210693
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,2560,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,1024,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,512,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,65536,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,12288,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,5120,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,7168,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,4096,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,8192,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,16384,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,10240,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,8192,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,12288,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,7168,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,10240,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,16384,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,4096,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,16384,65536,0.3235839903354645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,12288,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,10240,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,8192,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,7168,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,5120,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,16384,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,12288,65536,0.2457599937915802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,12288,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,8192,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,7168,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,10240,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,16384,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,3584,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,10240,65536,0.22323200106620789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,5120,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,8192,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,12288,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,7168,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,8192,65536,0.1884160041809082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,10240,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,16384,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,2048,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,12288,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,8192,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,10240,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,7168,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,16384,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,5120,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,4096,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,2048,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,256,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,7168,65536,0.18534399569034576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,12288,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,5120,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,16384,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,2048,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,5120,65536,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,10240,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,8192,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,16384,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,7168,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,4096,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,4096,65536,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,1024,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,12288,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,10240,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,7168,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,16384,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,8192,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3584,65536,0.12902399897575378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,3072,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,2560,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,1024,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,2048,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,12288,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,10240,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,8192,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,16384,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,7168,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,5120,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,3072,65536,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,4096,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,2560,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,1024,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,3072,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,2048,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,12288,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,7168,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,16384,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,8192,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,10240,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2560,65536,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,4096,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,3584,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,3072,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,2560,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,2048,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,1536,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,12288,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,8192,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,2048,65536,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,7168,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,10240,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,5120,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,16384,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,3072,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,4096,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,2048,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,2560,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,1536,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,512,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,1024,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,12288,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,10240,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,16384,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,8192,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,7168,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1536,65536,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,3584,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,4096,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,2560,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,3072,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,2048,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,1536,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,512,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,8192,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,12288,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,7168,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,5120,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,16384,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,10240,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,1024,65536,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,3584,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,3072,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,2048,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,2560,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,1536,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,512,65536,0.1382399946451187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,12288,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,8192,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,10240,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,7168,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,5120,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,4096,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,16384,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,3072,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,2560,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,3584,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,2048,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,1536,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,12288,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,5120,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,7168,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,16384,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,10240,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,256,65536,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,4096,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,3584,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,3072,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,512,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,1536,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,2048,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,4096,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,5120,0.11264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,2,128,65536,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,8192,0.17715199291706085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,7168,0.1454080045223236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,10240,0.20070399343967438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,3072,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,3584,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,12288,0.23552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,2560,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,512,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,1024,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,2048,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,16384,0.3338240087032318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,65536,1536,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,12288,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,4096,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,5120,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,7168,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,16384,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,8192,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,10240,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,3584,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,2560,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,3072,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,12288,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,8192,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,5120,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,7168,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,10240,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,16384,0.07782399654388428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,4096,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,12288,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,8192,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,7168,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,16384,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,10240,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,16384,65536,0.3317759931087494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,12288,65536,0.2611199915409088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,12288,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,10240,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,8192,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,7168,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,4096,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,5120,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,16384,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,10240,65536,0.22732800245285034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,12288,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,5120,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,7168,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,10240,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,8192,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,16384,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,8192,65536,0.19046400487422943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,7168,65536,0.2099200040102005
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,12288,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,8192,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,10240,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,7168,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,5120,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,16384,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,12288,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,8192,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,7168,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,5120,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,16384,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,10240,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,2048,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,5120,65536,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,128,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,4096,65536,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,12288,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,8192,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,10240,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,16384,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,2048,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,12288,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,8192,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,10240,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,2048,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3584,65536,0.12800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,128,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,12288,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,8192,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,16384,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,10240,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,3072,65536,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,2048,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,12288,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2560,65536,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,16384,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,10240,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,7168,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,8192,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,2048,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,8192,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,12288,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,16384,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,10240,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,2048,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,2048,65536,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,1024,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,10240,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,12288,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,5120,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,7168,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1536,65536,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,16384,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,3584,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,4096,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,2048,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,12288,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,7168,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,8192,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,10240,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,16384,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,1024,65536,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,3584,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,2048,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,1024,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,512,65536,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,12288,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,16384,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,5120,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,7168,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,10240,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,2048,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,12288,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,5120,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,7168,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,10240,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,16384,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,256,65536,0.12185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,3072,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,1024,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,2048,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0,int4_wo,1,128,65536,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,3584,7.400447845458984
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,4096,8.30361557006836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,5120,10.124287605285645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,2560,5.581823825836182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,3072,6.4983038902282715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,7168,13.722623825073242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,2048,4.658175945281982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,512,1.8380800485610962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,8192,15.5217924118042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,256,1.2840960025787354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,1024,2.8487679958343506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,128,1.031167984008789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,1536,3.752959966659546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,10240,19.20921516418457
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,8192,3.891200065612793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,10240,4.802559852600098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,12288,22.796287536621094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,12288,5.705728054046631
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,7168,3.4600958824157715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,5120,2.5425920486450195
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,4096,2.08076810836792
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,3072,1.632256031036377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,16384,7.532544136047363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,3584,1.8595839738845825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,2560,1.410048007965088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,512,0.46694400906562805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,128,0.2672640085220337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,256,0.32767999172210693
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,1024,0.7249919772148132
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,1536,0.9512959718704224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,2048,1.176576018333435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,7168,2.6275839805603027
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,8192,2.9675519466400146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,10240,3.668992042541504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,12288,4.355072021484375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,4096,1.5902719497680664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,65536,16384,30.035968780517578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,5120,1.9394559860229492
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,3584,1.4202879667282104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,3072,1.2472319602966309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,1536,0.7270399928092957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,2048,0.9011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,2560,1.0741759538650513
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,1024,0.5519359707832336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,256,0.24985599517822266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,16384,5.741568088531494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,512,0.35225600004196167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,128,0.20070399343967438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,7168,2.181119918823242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,8192,2.4586238861083984
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,10240,3.0443520545959473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,12288,3.6085760593414307
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,4096,1.3189120292663574
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,5120,1.6128000020980835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,3584,1.1816960573196411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,3072,1.035264015197754
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,2048,0.749567985534668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,2560,0.8908799886703491
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,16384,4.7421441078186035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,1536,0.6072319746017456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,256,0.2088959962129593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,512,0.2959359884262085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,128,0.16896000504493713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,1024,0.45977601408958435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,8192,1.9517439603805542
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,10240,2.407423973083496
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,12288,2.8631041049957275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,5120,1.2789759635925293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,7168,1.7274880409240723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,16384,3.763200044631958
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,4096,1.0506240129470825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,2048,0.596992015838623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,2560,0.7096319794654846
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,3072,0.822272002696991
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,3584,0.9349120259284973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,512,0.2385919988155365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,256,0.16998399794101715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,1024,0.36556801199913025
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,1536,0.48230400681495667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,128,0.13926400244235992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,10240,2.155519962310791
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,12288,2.564095973968506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,16384,3.3607680797576904
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,8192,1.7448960542678833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,7168,1.5472639799118042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,5120,1.1427839994430542
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,3584,0.8427519798278809
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,4096,0.936959981918335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,3072,0.7362560033798218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,2048,0.5335040092468262
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,2560,0.6369280219078064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,1536,0.43110400438308716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,512,0.21094399690628052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,1024,0.32972800731658936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,256,0.14950400590896606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,128,0.1228799968957901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,12288,1.8124799728393555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,16384,2.37772798538208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,10240,1.5267839431762695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,8192,1.235967993736267
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,7168,1.0987520217895508
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,5120,0.8089600205421448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,3584,0.5959680080413818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,4096,0.6676480174064636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,12288,65536,25.609216690063477
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,2560,0.4536319971084595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,3072,0.5232639908790588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,2048,0.37990400195121765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,16384,65536,34.03366470336914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,1024,0.23449599742889404
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,512,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,1536,0.3092480003833771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,256,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,128,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,10240,65536,21.25312042236328
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,8192,65536,16.866304397583008
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,10240,1.269760012626648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,12288,1.5073280334472656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,8192,1.0301439762115479
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,7168,0.9154559969902039
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,5120,0.6737920045852661
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,3584,0.49561598896980286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,16384,1.985535979270935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,4096,0.5560320019721985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,1024,0.1955839991569519
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,1536,0.25804799795150757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,2048,0.3164159953594208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,2560,0.3768320083618164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,512,0.12390399724245071
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,256,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,128,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,3072,0.4362240135669708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,8192,0.9297919869422913
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,7168,65536,14.732288360595703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,10240,1.1448320150375366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,12288,1.3598719835281372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,4096,0.5007359981536865
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,5120,0.6082559823989868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,3584,0.4556800127029419
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,7168,0.8243200182914734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,16384,1.7827839851379395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,1024,0.17510400712490082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,1536,0.23244799673557281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,2048,0.28672000765800476
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,3072,0.39526399970054626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,2560,0.3389439880847931
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,512,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,256,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,128,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,5120,65536,10.513407707214355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,8192,0.8273919820785522
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,10240,1.0199040174484253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,12288,1.2072960138320923
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,7168,0.7311360239982605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,5120,0.5427200198173523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,4096,0.44646400213241577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,16384,1.58515202999115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,3584,0.5335040092468262
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,3072,0.35020801424980164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,2560,0.3051519989967346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,2048,0.2549760043621063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,1024,0.15462400019168854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,1536,0.20684799551963806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,512,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,256,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,128,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,8192,0.6277120113372803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,10240,0.7731199860572815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,12288,0.9113600254058838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,5120,0.4116480052471161
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,7168,0.5550079941749573
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,16384,1.1970560550689697
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,4096,0.3389439880847931
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,2048,0.1955839991569519
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,2560,0.23244799673557281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,3072,0.2693119943141937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,3584,0.3041279911994934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,256,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,512,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,1024,0.12185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,1536,0.159743994474411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,128,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,4096,65536,8.535039901733398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,10240,0.6420480012893677
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,12288,0.7577599883079529
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,8192,0.5212159752845764
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,16384,0.997376024723053
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,5120,0.3420160114765167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,7168,0.4618239998817444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,4096,0.28467199206352234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,3584,0.2529279887676239
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,3072,0.22323200106620789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,2560,0.1945600062608719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,1536,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,2048,0.16281600296497345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,512,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,1024,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,256,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,128,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3584,65536,7.670783996582031
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,8192,0.41676801443099976
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,10240,0.5150719881057739
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,12288,0.6072319746017456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,16384,0.7987200021743774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,5120,0.2754560112953186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,4096,0.22835199534893036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,7168,0.37068799138069153
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,3072,65536,6.636544227600098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,2560,0.15564799308776855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,3072,0.1802240014076233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,3584,0.20377600193023682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,2048,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,256,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,512,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,128,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,1024,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,1536,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,8192,0.31436800956726074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,10240,0.38707199692726135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,12288,0.45772799849510193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2560,65536,5.285888195037842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,16384,0.6010879874229431
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,5120,0.20684799551963806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,4096,0.17203199863433838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,7168,0.27955201268196106
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,1536,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,2048,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,3584,0.15462400019168854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,2560,0.11878400295972824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,3072,0.1382399946451187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,256,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,128,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,512,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,1024,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,8192,0.17919999361038208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,10240,0.22937600314617157
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,12288,0.26419198513031006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,2048,65536,4.360191822052002
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,5120,0.12185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,16384,0.38809600472450256
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,7168,0.17715199291706085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,4096,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,3584,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,1536,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,2048,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,2560,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,3072,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,1024,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,512,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,128,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,256,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,8192,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,10240,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,12288,0.158720001578331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1536,65536,3.379199981689453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,16384,0.20684799551963806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,7168,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,5120,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,4096,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,3584,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,1536,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,2048,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,2560,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,3072,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,1024,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,8192,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,10240,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,1024,65536,2.6030080318450928
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,12288,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,16384,0.14745600521564484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,3584,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,5120,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,7168,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,4096,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,1536,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,2048,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,2560,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,3072,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,512,65536,1.6650240421295166
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,256,65536,0.7813119888305664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8192,128,65536,0.5058559775352478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,5120,5.081088066101074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,4096,4.15231990814209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,7168,6.8679680824279785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,3584,3.714047908782959
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,8192,7.767039775848389
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,3072,3.265536069869995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,2560,2.802687883377075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,2048,2.3408639430999756
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,1024,1.427456021308899
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,1536,1.8872319459915161
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,256,0.647167980670929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,512,0.9154559969902039
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,128,0.5150719881057739
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,10240,9.607168197631836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,12288,11.410431861877441
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,7168,1.7285120487213135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,8192,1.9486720561981201
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,10240,2.4145920276641846
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,12288,2.861056089401245
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,5120,1.277951955795288
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,4096,1.0465279817581177
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,3584,0.936959981918335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,3072,0.8253440260887146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,1024,0.36454400420188904
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,1536,0.48025599122047424
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,2048,0.5949440002441406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,2560,0.7086079716682434
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,512,0.2396160066127777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,128,0.1382399946451187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,256,0.17100800573825836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,16384,3.782655954360962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,7168,1.3619199991226196
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,65536,16384,15.032320022583008
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,8192,1.5400960445404053
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,10240,1.903615951538086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,12288,2.2599680423736572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,4096,0.8294399976730347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,3584,0.7413759827613831
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,5120,1.0096640586853027
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,3072,0.6502400040626526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,2560,0.5611519813537598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,1536,0.37785598635673523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,1024,0.28467199206352234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,2048,0.4700160026550293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,512,0.1802240014076233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,128,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,256,0.12902399897575378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,16384,2.9655039310455322
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,7168,1.094655990600586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,8192,1.2380160093307495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,10240,1.5247360467910767
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,12288,1.8083839416503906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,4096,0.6656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,5120,0.8089600205421448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,3584,0.5949440002441406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,3072,0.5222399830818176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,2048,0.37785598635673523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,2560,0.45260798931121826
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,16384,2.382848024368286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,1536,0.3051519989967346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,512,0.15564799308776855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,128,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,1024,0.23244799673557281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,256,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,8192,1.031167984008789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,10240,1.2718080282211304
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,12288,1.6834559440612793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,7168,0.9134079813957214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,5120,0.6758400201797485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,16384,1.9793920516967773
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,4096,0.5560320019721985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,2048,0.317440003156662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,2560,0.3768320083618164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,3072,0.4352000057697296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,3584,0.4986880123615265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,512,0.12595200538635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,256,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,1024,0.1945600062608719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,1536,0.25600001215934753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,128,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,10240,1.1458560228347778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,12288,1.3608959913253784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,16384,1.7827839851379395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,8192,0.9267200231552124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,7168,0.8232960104942322
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,5120,0.6082559823989868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,3584,0.449535995721817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,4096,0.4997119903564453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,3072,0.39321601390838623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,2048,0.28569599986076355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,2560,0.3420160114765167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,1536,0.2303999960422516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,512,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,1024,0.17203199863433838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,256,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,128,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,12288,0.9123839735984802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,16384,1.1950080394744873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,10240,0.7690240144729614
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,8192,0.6225919723510742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,12288,65536,12.894207954406738
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,5120,0.40959998965263367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,7168,0.5539839863777161
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,4096,0.33792001008987427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,16384,65536,16.680959701538086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,3072,0.2672640085220337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,3584,0.3031040132045746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,10240,65536,10.583040237426758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,2560,0.2314240038394928
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,2048,0.1945600062608719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,512,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,1536,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,256,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,1024,0.12083200365304947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,128,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,8192,65536,8.60364818572998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,8192,0.52019202709198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,10240,0.6399999856948853
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,12288,0.7567359805107117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,7168,0.4607999920845032
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,5120,0.3420160114765167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,4096,0.2826240062713623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,3584,0.2539519965648651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,16384,0.9994239807128906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,3072,0.22220799326896667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,2560,0.19353599846363068
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,2048,0.16281600296497345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,512,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,1024,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,1536,0.13209599256515503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,256,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,128,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,7168,65536,7.69536018371582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,8192,0.5191680192947388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,10240,0.6399999856948853
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,12288,0.7567359805107117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,5120,0.3409920036792755
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,4096,0.2815999984741211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,7168,0.4607999920845032
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,3584,0.2519040107727051
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,16384,0.9953280091285706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,1024,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,1536,0.12595200538635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,3072,0.22220799326896667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,2048,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,2560,0.19148799777030945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,128,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,256,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,512,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,5120,65536,5.268479824066162
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,8192,0.4188160002231598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,10240,0.5140479803085327
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,12288,0.6082559823989868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,7168,0.3696640133857727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,5120,0.2764799892902374
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,4096,0.22732800245285034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,16384,0.7997440099716187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,3584,0.20479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,2560,0.158720001578331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,1536,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,3072,0.1812479943037033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,2048,0.131071999669075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,1024,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,128,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,256,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,512,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,8192,0.317440003156662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,10240,0.3891200125217438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,12288,0.45977601408958435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,16384,0.6031360030174255
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,5120,0.20787200331687927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,7168,0.2815999984741211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,4096,0.1740799993276596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,3584,0.15667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,2048,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,2560,0.12083200365304947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,3072,0.1382399946451187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,512,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,256,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,1024,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,1536,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,128,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,4096,65536,4.343808174133301
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,8192,0.31539198756217957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,10240,0.3891200125217438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,12288,0.45772799849510193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,16384,0.609279990196228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,4096,0.1730560064315796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,5120,0.2088959962129593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,7168,0.27955201268196106
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3584,65536,4.168704032897949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,2560,0.11673600226640701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,3072,0.13619199395179749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,2048,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,3584,0.15462400019168854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,1536,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,128,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,256,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,512,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,1024,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,8192,0.21401600539684296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,10240,0.26214399933815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,12288,0.3092480003833771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,3072,65536,3.3720319271087646
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,16384,0.4034560024738312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,5120,0.14131200313568115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,7168,0.19046400487422943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,4096,0.11878400295972824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,3584,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,1536,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,1024,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,2048,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,3072,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,2560,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,512,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,128,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,256,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2560,65536,2.6613759994506836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,8192,0.19251200556755066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,10240,0.22732800245285034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,12288,0.26419198513031006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,5120,0.12083200365304947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,16384,0.37273600697517395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,7168,0.1802240014076233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,4096,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,3584,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,1536,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,2048,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,2560,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,1024,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,3072,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,256,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,512,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,128,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,8192,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,2048,65536,2.4934399127960205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,10240,0.13619199395179749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,12288,0.158720001578331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,16384,0.19865599274635315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,4096,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,3584,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,5120,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,7168,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,3072,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,1024,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,1536,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,2560,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,2048,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1536,65536,1.7274880409240723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,8192,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,10240,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,12288,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,5120,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,16384,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,7168,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,4096,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,3584,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,3072,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,1024,65536,1.6250879764556885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,8192,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,10240,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,12288,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,7168,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,16384,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,5120,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,4096,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,3584,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,3072,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,512,65536,0.78438401222229
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,256,65536,0.4864000082015991
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4096,128,65536,0.2682879865169525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,5120,2.540544033050537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,4096,2.086911916732788
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,7168,3.441663980484009
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,3584,1.8595839738845825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,8192,3.8901760578155518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,3072,1.6373759508132935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,2560,1.4069759845733643
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,2048,1.171455979347229
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,1024,0.7188479900360107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,512,0.4689919948577881
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,256,0.33484798669815063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,1536,0.9451519846916199
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,10240,4.814847946166992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,128,0.26521599292755127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,12288,5.708799839019775
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,7168,0.9123839735984802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,8192,1.031167984008789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,10240,1.2707840204238892
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,12288,1.511423945426941
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,5120,0.6758400201797485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,4096,0.5529599785804749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,3584,0.49561598896980286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,3072,0.4341759979724884
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,2560,0.3758080005645752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,2048,0.31436800956726074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,1024,0.1945600062608719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,512,0.12595200538635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,1536,0.2539519965648651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,256,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,16384,1.9845119714736938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,128,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,65536,16384,7.5141119956970215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,7168,0.7290880084037781
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,8192,0.826367974281311
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,10240,1.018880009651184
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,12288,1.2072960138320923
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,5120,0.5406720042228699
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,4096,0.4433920085430145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,3584,0.3983359932899475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,3072,0.35020801424980164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,2560,0.30003198981285095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,2048,0.2549760043621063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,1536,0.2017280012369156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,1024,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,512,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,16384,1.58515202999115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,256,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,128,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,7168,0.5529599785804749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,8192,0.6215680241584778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,10240,0.7659519910812378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,12288,0.9113600254058838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,4096,0.33689600229263306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,5120,0.40959998965263367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,3584,0.30003198981285095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,3072,0.26521599292755127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,16384,1.201151967048645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,2560,0.22835199534893036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,2048,0.19148799777030945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,1536,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,512,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,1024,0.12083200365304947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,256,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,128,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,8192,0.5191680192947388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,10240,0.638975977897644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,12288,0.7608320116996765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,16384,1.0004479885101318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,5120,0.3409920036792755
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,4096,0.2815999984741211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,7168,0.45977601408958435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,2048,0.16076800227165222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,2560,0.19148799777030945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,3072,0.22118400037288666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,3584,0.25088000297546387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,512,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,1024,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,256,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,1536,0.131071999669075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,128,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,10240,0.638975977897644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,12288,0.7587839961051941
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,16384,0.9943040013313293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,7168,0.45977601408958435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,8192,0.5181440114974976
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,5120,0.3420160114765167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,3584,0.24985599517822266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,4096,0.2805759906768799
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,3072,0.22118400037288666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,2048,0.15564799308776855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,2560,0.187391996383667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,1536,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,512,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,1024,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,256,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,128,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,12288,0.45875200629234314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,12288,65536,6.540287971496582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,16384,65536,8.555520057678223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,16384,0.6041600108146667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,8192,0.3164159953594208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,10240,65536,5.268479824066162
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,5120,0.2088959962129593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,10240,0.38707199692726135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,7168,0.27955201268196106
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,3584,0.15564799308776855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,2048,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,4096,0.17203199863433838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,2560,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,3072,0.13619199395179749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,256,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,512,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,128,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,1024,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,1536,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,8192,65536,4.299776077270508
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,7168,0.27955201268196106
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,8192,0.31539198756217957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,10240,0.38604798913002014
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,12288,0.45875200629234314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,4096,0.17100800573825836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,5120,0.20787200331687927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,3584,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,3072,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,16384,0.6000639796257019
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,512,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,1536,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,2560,0.11673600226640701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,2048,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,1024,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,256,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,128,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,7168,65536,4.150271892547607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,8192,0.29183998703956604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,12288,0.4403199851512909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,10240,0.37785598635673523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,7168,0.2611199915409088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,5120,0.19251200556755066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,16384,0.5990399718284607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,3584,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,4096,0.1525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,3072,0.12083200365304947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,1024,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,1536,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,2048,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,2560,0.10342399775981903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,256,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,512,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,128,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,5120,65536,2.6664960384368896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,8192,0.21299199759960175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,10240,0.26214399933815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,12288,0.3092480003833771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,16384,0.4034560024738312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,5120,0.14131200313568115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,7168,0.18943999707698822
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,4096,0.11673600226640701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,3584,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,2048,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,2560,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,3072,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,1536,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,1024,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,256,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,512,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,128,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,8192,0.20787200331687927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,10240,0.26009601354599
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,12288,0.30822399258613586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,4096,65536,2.4883201122283936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,16384,0.4034560024738312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,4096,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,3584,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,7168,0.1884160041809082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,5120,0.1382399946451187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,3072,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,1024,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,2560,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,1536,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,2048,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,512,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,256,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,128,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3584,65536,2.322432041168213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,8192,0.19046400487422943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,12288,0.26316800713539124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,10240,0.22323200106620789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,16384,0.35839998722076416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,5120,0.11878400295972824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,7168,0.16076800227165222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,3584,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,4096,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,3072,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,2560,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,1024,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,1536,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,2048,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,512,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,256,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,128,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,3072,65536,1.7643519639968872
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,8192,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,7168,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,10240,0.13619199395179749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,12288,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,16384,0.21196800470352173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,4096,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,3584,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,5120,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,3072,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,512,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,2048,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,1024,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,1536,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,2560,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,256,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,128,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2560,65536,1.6506880521774292
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,8192,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,7168,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,12288,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,10240,0.131071999669075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,16384,0.19251200556755066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,4096,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,5120,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,3072,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,3584,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,2560,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,1024,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,1536,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,2048,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,256,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,128,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,2048,65536,1.6087039709091187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,8192,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,12288,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,10240,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,7168,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,16384,0.13414399325847626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,4096,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,5120,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,2560,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1536,65536,1.0280959606170654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,7168,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,8192,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,10240,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,12288,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,16384,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,5120,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,4096,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,1024,65536,0.78438401222229
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,7168,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,512,65536,0.48127999901771545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,16384,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,2048,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,256,65536,0.2672640085220337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2048,128,65536,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,5120,1.274880051612854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,4096,1.0485759973526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,7168,1.7326079607009888
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,8192,1.9496959447860718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,3584,0.9349120259284973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,3072,0.8212479948997498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,2560,0.7096319794654846
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,2048,0.5908480286598206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,1024,0.36454400420188904
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,512,0.24371199309825897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,256,0.17715199291706085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,1536,0.4792320132255554
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,10240,2.411520004272461
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,128,0.14028799533843994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,12288,2.8620800971984863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,8192,0.5191680192947388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,7168,0.4618239998817444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,10240,0.638975977897644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,5120,0.3409920036792755
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,12288,0.7608320116996765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,4096,0.2815999984741211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,3584,0.2529279887676239
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,2048,0.16281600296497345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,2560,0.19251200556755066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,3072,0.22118400037288666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,1024,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,1536,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,16384,0.9994239807128906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,256,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,512,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,128,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,65536,16384,3.7570559978485107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,7168,0.3696640133857727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,8192,0.4177919924259186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,5120,0.2734079957008362
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,10240,0.5130239725112915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,12288,0.6082559823989868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,4096,0.22630399465560913
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,3584,0.2027519941329956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,3072,0.17919999361038208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,2560,0.15462400019168854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,2048,0.131071999669075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,512,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,1024,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,1536,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,16384,0.7997440099716187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,256,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,128,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,7168,0.27955201268196106
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,8192,0.31436800956726074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,10240,0.38809600472450256
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,12288,0.45772799849510193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,5120,0.20787200331687927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,4096,0.17203199863433838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,3584,0.15462400019168854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,3072,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,16384,0.6021119952201843
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,512,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,2560,0.12185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,2048,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,1024,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,1536,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,128,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,256,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,8192,0.31539198756217957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,12288,0.45875200629234314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,10240,0.38604798913002014
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,16384,0.6000639796257019
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,4096,0.17100800573825836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,5120,0.2088959962129593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,7168,0.27955201268196106
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,3584,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,3072,0.13619199395179749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,2560,0.11878400295972824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,2048,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,1536,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,1024,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,256,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,512,0.23244799673557281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,128,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,12288,0.4331519901752472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,10240,0.374783992767334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,16384,0.6010879874229431
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,7168,0.26214399933815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,8192,0.2908160090446472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,5120,0.19353599846363068
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,3584,0.1382399946451187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,4096,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,16384,65536,4.317183971405029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,3072,0.12185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,12288,65536,3.4017279148101807
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,1536,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,2560,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,1024,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,2048,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,512,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,256,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,128,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,10240,65536,2.6716160774230957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,8192,0.20479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,10240,0.26009601354599
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,12288,0.30822399258613586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,7168,0.187391996383667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,16384,0.4106239974498749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,5120,0.29286399483680725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,4096,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,3072,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,3584,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,1536,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,2560,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,8192,65536,2.515968084335327
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,1024,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,2048,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,512,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,256,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,128,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,7168,0.1740799993276596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,12288,0.2590720057487488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,8192,0.17612800002098083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,10240,0.22220799326896667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,16384,0.3604480028152466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,3072,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,3584,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,4096,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,5120,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,1024,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,2560,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,1536,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,512,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,2048,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,256,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,128,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,7168,65536,2.323456048965454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,8192,0.16998399794101715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,7168,0.16896000504493713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,12288,0.2519040107727051
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,10240,0.21606400609016418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,16384,0.3481599986553192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,5120,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,4096,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,3072,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,3584,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,512,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,2560,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,2048,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,1024,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,1536,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,256,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,128,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,5120,65536,1.6486400365829468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,12288,0.158720001578331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,8192,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,7168,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,10240,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,16384,0.20684799551963806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,5120,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,4096,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,3584,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,3072,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,2560,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,1024,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,2048,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,512,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,1536,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,256,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,128,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,4096,65536,1.6168960332870483
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,8192,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,10240,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,12288,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,7168,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,16384,0.20582400262355804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,4096,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,3584,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,3072,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,5120,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,2560,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3584,65536,1.4489599466323853
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,1024,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,2048,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,1536,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,7168,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,8192,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,10240,0.13414399325847626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,12288,0.14745600521564484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,3072,65536,0.9308159947395325
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,4096,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,16384,0.19968000054359436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,3072,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,5120,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,3584,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,2560,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,1024,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,2048,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,1536,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,8192,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,7168,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,10240,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,12288,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,3072,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2560,65536,0.8458240032196045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,4096,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,3584,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,16384,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,5120,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,2048,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,8192,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,7168,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,12288,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,10240,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,16384,0.13619199395179749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,4096,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,5120,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,2048,65536,0.7813119888305664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,2560,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,7168,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,8192,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,12288,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,10240,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1536,65536,0.7290880084037781
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,16384,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,3584,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,5120,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,4096,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,2048,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,1024,65536,0.477183997631073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,12288,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,2048,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,512,65536,0.26316800713539124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,7168,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,8192,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,5120,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,16384,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,2048,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,256,65536,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,4096,0.826367974281311
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1024,128,65536,0.1525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,5120,1.0096640586853027
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,3584,0.7393280267715454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,3072,0.6502400040626526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,7168,1.3629440069198608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,2048,0.4689919948577881
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,8192,1.5400960445404053
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,1536,0.37785598635673523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,2560,0.562175989151001
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,1024,0.2887679934501648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,512,0.19046400487422943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,256,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,128,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,12288,0.609279990196228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,10240,1.9005440473556519
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,8192,0.41676801443099976
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,12288,2.2599680423736572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,5120,0.2744320034980774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,10240,0.5120000243186951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,7168,0.37068799138069153
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,2048,0.131071999669075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,3584,0.2027519941329956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,4096,0.22630399465560913
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,3072,0.17817600071430206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,2560,0.15462400019168854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,256,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,1536,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,128,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,512,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,1024,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,16384,0.7976959943771362
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,7168,0.27955201268196106
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,8192,0.31539198756217957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,65536,16384,2.9675519466400146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,5120,0.2088959962129593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,10240,0.38707199692726135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,12288,0.45772799849510193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,3072,0.13619199395179749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,4096,0.1730560064315796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,3584,0.15462400019168854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,2560,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,2048,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,1536,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,256,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,512,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,16384,0.6010879874229431
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,128,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,1024,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,7168,0.27955201268196106
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,8192,0.31436800956726074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,10240,0.38707199692726135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,12288,0.45772799849510193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,4096,0.17100800573825836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,5120,0.20787200331687927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,3584,0.15462400019168854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,16384,0.6000639796257019
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,3072,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,2048,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,2560,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,512,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,1536,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,1024,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,256,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,128,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,8192,0.21196800470352173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,10240,0.2611199915409088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,12288,0.30822399258613586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,5120,0.14336000382900238
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,16384,0.4044800102710724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,4096,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,7168,0.19046400487422943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,3584,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,3072,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,2560,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,2048,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,65536,1.768447995185852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,1536,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,512,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,256,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,1024,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,8192,128,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,8192,0.2529279887676239
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,10240,0.3051519989967346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,12288,0.35942399501800537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,5120,0.15667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,16384,0.40243199467658997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,4096,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,7168,0.21196800470352173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,3584,0.12083200365304947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,3072,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,2560,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,2048,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,1536,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,1024,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,512,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,256,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,128,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,16384,65536,3.4242560863494873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,12288,65536,2.5763840675354004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,12288,0.30720001459121704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,10240,0.26009601354599
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,16384,0.40243199467658997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,7168,0.187391996383667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,8192,0.19046400487422943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,4096,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,5120,0.11468800157308578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,10240,65536,2.442239999771118
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,3584,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,2048,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,2560,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,3072,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,1024,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,1536,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,256,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,512,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,128,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,7168,0.1382399946451187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,8192,0.13926400244235992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,12288,0.24883200228214264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,10240,0.19968000054359436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,4096,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,3072,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,3584,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,5120,0.11264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,16384,0.2949120104312897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,2560,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,512,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,2048,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,1024,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,256,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,1536,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,128,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,7168,65536,1.9322880506515503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,8192,0.16281600296497345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,10240,0.19046400487422943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,12288,0.22732800245285034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,16384,0.27238398790359497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,3584,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,4096,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,5120,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,7168,0.1382399946451187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,2560,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,1024,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,3072,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,2048,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,1536,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,512,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,256,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,5120,65536,1.575935959815979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,128,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,7168,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,8192,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,10240,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,12288,0.15667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,16384,0.20582400262355804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,3072,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,3584,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,4096,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,5120,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,2048,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,1536,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,2560,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,4096,65536,0.9359359741210938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,512,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,256,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,128,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,7168,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,8192,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,12288,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,10240,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,16384,0.20684799551963806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,4096,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,3072,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,3584,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,5120,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3584,65536,1.1806720495224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,512,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,2048,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,1024,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,2560,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,1536,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,8192,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,7168,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,12288,0.11468800157308578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,10240,0.13414399325847626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,5120,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,16384,0.1525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,4096,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,3584,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,3072,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,3072,65536,0.7864320278167725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,8192,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,7168,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,12288,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,10240,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,5120,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2560,65536,0.7915520071983337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,16384,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,4096,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,3584,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,3072,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,8192,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,10240,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,12288,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,7168,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,2048,65536,0.7782400250434875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,5120,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,16384,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,3584,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,4096,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,3072,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,12288,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,7168,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,10240,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,16384,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1536,65536,0.6594560146331787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,5120,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,1024,65536,0.37785598635673523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,2048,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,5120,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,512,65536,0.19046400487422943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,16384,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,5120,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,2048,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,1536,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,256,65536,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,768,128,65536,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,4096,0.5539839863777161
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,5120,0.6737920045852661
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,7168,0.9134079813957214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,3584,0.49459201097488403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,8192,1.031167984008789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,2560,0.37887999415397644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,3072,0.43724799156188965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,256,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,1024,0.19968000054359436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,10240,1.2728320360183716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,512,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,2048,0.3164159953594208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,1536,0.25702399015426636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,128,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,12288,1.5083520412445068
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,5120,0.20787200331687927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,7168,0.27852800488471985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,8192,0.31334400177001953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,10240,0.38604798913002014
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,12288,0.45772799849510193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,3072,0.13619199395179749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,3584,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,4096,0.17100800573825836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,65536,16384,1.9814399480819702
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,16384,0.6010879874229431
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,2048,0.10342399775981903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,2560,0.11980800330638885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,512,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,1024,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,256,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,128,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,1536,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,5120,0.14131200313568115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,7168,0.19046400487422943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,8192,0.21196800470352173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,10240,0.26009601354599
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,12288,0.30822399258613586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,4096,0.11673600226640701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,3584,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,3072,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,2560,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,2048,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,16384,0.40243199467658997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,512,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,1024,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,1536,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,128,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,256,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,7168,0.1884160041809082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,8192,0.20684799551963806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,12288,0.30822399258613586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,10240,0.26009601354599
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,16384,0.4034560024738312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,5120,0.14028799533843994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,4096,0.11366400122642517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,3584,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,3072,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,2560,0.07782399654388428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,1024,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,512,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,1536,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,2048,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,128,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,256,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,8192,0.17919999361038208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,12288,0.2662400007247925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,10240,0.22732800245285034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,16384,0.37785598635673523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,5120,0.11980800330638885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,4096,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,7168,0.17510400712490082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,3584,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,3072,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,2560,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,2048,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,1536,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,256,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,1024,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,512,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,128,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,16384,65536,2.4698879718780518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,12288,65536,1.7571840286254883
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,12288,0.25600001215934753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,10240,0.2170879989862442
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,8192,0.17203199863433838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,16384,0.35020801424980164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,4096,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,5120,0.11468800157308578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,7168,0.15462400019168854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,3584,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,3072,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,1536,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,2048,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,1024,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,2560,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,512,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,10240,65536,1.6558079719543457
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,128,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,256,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,7168,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,8192,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,12288,0.158720001578331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,10240,0.13414399325847626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,16384,0.20582400262355804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,3584,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,4096,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,5120,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,3072,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,8192,65536,1.6117759943008423
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,2560,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,1536,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,2048,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,1024,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,512,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,7168,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,8192,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,10240,0.13414399325847626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,12288,0.158720001578331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,16384,0.20479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,3584,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,3072,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,5120,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,4096,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,2560,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,7168,65536,1.4417920112609863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,2048,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,1536,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,512,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,1024,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,7168,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,12288,0.1443839967250824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,5120,65536,0.7905279994010925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,8192,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,10240,0.13414399325847626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,16384,0.18943999707698822
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,3584,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,4096,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,3072,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,2560,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,5120,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,2048,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,1536,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,1024,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,12288,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,7168,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,4096,65536,0.7823359966278076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,8192,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,10240,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,16384,0.14745600521564484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,3584,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,3072,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,4096,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,5120,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,1024,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,12288,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,8192,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,7168,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,10240,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3584,65536,0.7854080200195312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,16384,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,5120,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,3072,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,3584,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,4096,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,3072,65536,0.7147520184516907
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,12288,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,10240,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,7168,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,8192,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,5120,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,16384,0.14233599603176117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,4096,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,2048,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,2560,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,1024,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2560,65536,0.5631999969482422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,7168,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,8192,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,12288,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,10240,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,5120,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,16384,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,3584,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,3072,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,4096,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,8192,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,12288,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,7168,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,2048,65536,0.48230400681495667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,10240,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,16384,0.07782399654388428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,4096,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,5120,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,3584,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1536,65536,0.30822399258613586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,8192,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,16384,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,5120,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,2048,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,1536,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,1024,65536,0.30822399258613586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,7168,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,8192,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,2560,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,512,65536,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,8192,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,256,65536,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,512,128,65536,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,3584,0.4915199875831604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,5120,0.6768640279769897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,4096,0.5519359707832336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,2048,0.31334400177001953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,7168,0.7321599721908569
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,1024,0.1955839991569519
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,8192,0.8253440260887146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,3072,0.43212801218032837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,2560,0.37376001477241516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,256,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,1536,0.2549760043621063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,512,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,128,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,10240,1.264639973640442
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,8192,0.21196800470352173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,12288,1.210368037223816
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,7168,0.20070399343967438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,12288,0.3112959861755371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,5120,0.20582400262355804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,10240,0.26214399933815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,4096,0.16998399794101715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,2048,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,16384,0.4065279960632324
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,3584,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,3072,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,2560,0.11673600226640701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,1536,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,256,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,512,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,128,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,1024,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,65536,16384,1.5872000455856323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,10240,0.26009601354599
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,7168,0.18943999707698822
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,8192,0.21196800470352173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,4096,0.11878400295972824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,5120,0.14028799533843994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,12288,0.30822399258613586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,2560,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,2048,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,3584,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,1536,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,3072,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,256,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,512,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,16384,0.40243199467658997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,128,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,1024,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,7168,0.1884160041809082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,5120,0.14028799533843994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,8192,0.19763199985027313
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,10240,0.26009601354599
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,12288,0.30822399258613586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,4096,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,3584,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,2560,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,16384,0.4034560024738312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,2048,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,3072,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,1536,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,512,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,256,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,1024,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,128,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,12288,0.25702399015426636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,10240,65536,1.546239972114563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,8192,0.14028799533843994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,10240,0.2099200040102005
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,5120,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,7168,0.15564799308776855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,3584,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,16384,0.29183998703956604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,4096,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,2560,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,3072,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,2048,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,1536,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,1024,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,512,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,128,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,256,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,10240,0.19865599274635315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,8192,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,12288,0.2385919988155365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,16384,65536,2.3193600177764893
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,7168,0.12390399724245071
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,16384,0.26419198513031006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,3584,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,4096,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,5120,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,12288,65536,1.5523840188980103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,2048,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,3072,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,1536,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,2560,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,1024,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,512,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,256,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,128,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,8192,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,7168,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,12288,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,10240,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,5120,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,16384,0.20582400262355804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,4096,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,2560,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,3584,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,3072,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,1536,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,8192,65536,1.5452159643173218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,2048,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,8192,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,7168,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,12288,0.158720001578331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,10240,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,5120,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,16384,0.20684799551963806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,4096,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,3584,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,3072,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,2560,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,1536,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,1024,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,7168,65536,1.1161600351333618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,8192,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,7168,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,12288,0.11878400295972824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,10240,0.13414399325847626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,5120,65536,0.7782400250434875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,4096,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,5120,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,3584,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,3072,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,16384,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,2560,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,1024,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,1536,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,12288,0.10342399775981903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,8192,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,7168,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,16384,0.1382399946451187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,10240,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,4096,65536,0.8089600205421448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,5120,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,3072,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,3584,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,4096,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,1536,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,1024,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,7168,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,8192,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,10240,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,12288,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,16384,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,4096,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,3584,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,5120,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3584,65536,0.7802879810333252
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,3072,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,1024,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,12288,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,10240,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,3072,65536,0.6635519862174988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,8192,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,7168,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,4096,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,16384,0.13414399325847626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,3584,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,3072,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,5120,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2560,65536,0.45977601408958435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,12288,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,7168,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,8192,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,10240,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,16384,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,5120,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,4096,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,3584,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,2048,65536,0.3624959886074066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,12288,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,7168,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,10240,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,16384,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,5120,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,2048,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,512,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1536,65536,0.29900801181793213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,5120,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,8192,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,2048,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,1024,65536,0.19968000054359436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,16384,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,512,65536,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,5120,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,8192,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,16384,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,256,65536,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,384,128,65536,0.14950400590896606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,4096,0.28467199206352234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,5120,0.35020801424980164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,7168,0.4679679870605469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,8192,0.5242879986763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,3072,0.22835199534893036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,3584,0.25702399015426636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,2560,0.19865599274635315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,10240,0.6512640118598938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,2048,0.1669120043516159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,1536,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,512,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,1024,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,256,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,128,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,12288,0.7690240144729614
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,5120,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,7168,0.18943999707698822
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,8192,0.19660800695419312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,10240,0.2385919988155365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,65536,16384,1.006592035293579
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,12288,0.30003198981285095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,3584,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,4096,0.11264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,3072,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,2560,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,2048,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,16384,0.3891200125217438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,1536,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,128,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,1024,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,256,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,512,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,5120,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,8192,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,7168,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,12288,0.1658879965543747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,10240,0.13619199395179749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,16384,0.21401600539684296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,4096,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,3584,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,2048,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,3072,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,2560,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,1536,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,256,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,512,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,1024,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,128,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,7168,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,8192,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,10240,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,12288,0.159743994474411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,16384,0.21401600539684296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,3072,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,3584,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,4096,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,5120,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,2560,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,2048,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,1536,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,1024,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,512,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,256,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,128,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,12288,0.15667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,8192,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,10240,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,16384,0.20582400262355804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,16384,65536,1.6168960332870483
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,5120,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,4096,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,7168,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,12288,65536,0.92467200756073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,2560,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,3584,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,3072,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,2048,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,1024,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,1536,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,512,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,10240,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,8192,0.10342399775981903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,12288,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,7168,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,10240,65536,0.7915520071983337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,16384,0.20479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,5120,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,4096,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,3584,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,2560,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,2048,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,3072,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,1536,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,1024,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,8192,65536,0.7833600044250488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,12288,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,7168,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,10240,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,8192,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,5120,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,16384,0.14131200313568115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,4096,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,2560,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,2048,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,3072,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,3584,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,1536,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,1024,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,7168,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,8192,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,12288,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,16384,0.1382399946451187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,10240,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,7168,65536,0.7802879810333252
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,3584,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,3072,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,4096,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,2560,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,5120,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,1024,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,5120,65536,0.5775359869003296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,7168,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,8192,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,12288,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,10240,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,5120,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,16384,0.12697599828243256
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,3584,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,4096,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,4096,65536,0.4904960095882416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,7168,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,8192,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,12288,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,10240,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,16384,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,5120,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,3072,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,3584,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,4096,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3584,65536,0.4607999920845032
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,8192,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,7168,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,12288,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,10240,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,16384,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,4096,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,5120,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,3584,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,3072,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,3072,65536,0.33792001008987427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,12288,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,8192,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,10240,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,7168,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,16384,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,5120,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,4096,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2560,65536,0.30003198981285095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,12288,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,7168,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,10240,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,16384,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,4096,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,2048,65536,0.27033600211143494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,3584,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,10240,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1536,65536,0.20377600193023682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,5120,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,1024,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,1024,65536,0.15462400019168854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,8192,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,512,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,512,65536,0.14950400590896606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,7168,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,5120,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,10240,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,4096,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,3072,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,3584,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,1024,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,256,65536,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,256,128,65536,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,4096,0.2805759906768799
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,5120,0.34406399726867676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,7168,0.46489599347114563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,8192,0.5181440114974976
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,3584,0.2529279887676239
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,2560,0.19353599846363068
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,3072,0.22323200106620789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,10240,0.6430720090866089
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,2048,0.16486400365829468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,1536,0.13619199395179749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,512,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,1024,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,256,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,128,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,12288,0.7618560194969177
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,7168,0.1525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,5120,0.12390399724245071
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,8192,0.17203199863433838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,65536,16384,1.001471996307373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,10240,0.2303999960422516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,12288,0.27033600211143494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,4096,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,16384,0.3317759931087494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,1536,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,2560,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,3584,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,3072,0.07782399654388428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,2048,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,1024,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,512,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,128,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,256,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,10240,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,5120,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,7168,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,8192,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,12288,0.159743994474411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,4096,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,3584,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,2560,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,3072,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,1536,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,2048,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,16384,0.20582400262355804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,512,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,256,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,1024,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,128,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,7168,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,8192,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,12288,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,10240,0.1372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,16384,0.20479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,4096,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,5120,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,3072,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,3584,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,1536,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,512,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,2048,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,2560,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,1024,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,16384,0.20479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,12288,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,8192,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,10240,0.13926400244235992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,16384,65536,1.46943998336792
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,4096,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,5120,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,3584,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,7168,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,3072,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,2048,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,1536,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,2560,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,1024,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,12288,65536,0.7895039916038513
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,7168,0.07782399654388428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,12288,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,8192,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,10240,0.13414399325847626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,10240,65536,0.779263973236084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,5120,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,16384,0.20377600193023682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,4096,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,3072,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,3584,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,2560,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,2048,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,1024,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,1536,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,8192,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,8192,65536,0.7772160172462463
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,7168,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,12288,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,10240,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,16384,0.131071999669075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,5120,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,4096,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,3072,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,2560,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,3584,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,1536,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,1024,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,12288,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,10240,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,7168,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,8192,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,7168,65536,0.7772160172462463
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,16384,0.12902399897575378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,4096,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,5120,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,3584,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,512,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,5120,65536,0.4700160026550293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,16384,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,5120,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,7168,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,12288,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,8192,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,10240,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,4096,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,3584,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,3072,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,10240,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,4096,65536,0.3819519877433777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,7168,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,12288,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,8192,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,16384,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,5120,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,4096,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,3584,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,8192,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,7168,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,10240,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,12288,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3584,65536,0.3399679958820343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,16384,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,3584,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,5120,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,4096,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,3072,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,3072,65536,0.3409920036792755
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,12288,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,10240,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,7168,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,16384,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,4096,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,2048,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2560,65536,0.26521599292755127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,16384,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,1024,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,2048,65536,0.19763199985027313
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1536,65536,0.15667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,12288,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,1024,65536,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,12288,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,16384,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,4096,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,512,65536,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,5120,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,256,65536,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,8192,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,192,128,65536,0.14950400590896606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,4096,0.2815999984741211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,5120,0.34303998947143555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,7168,0.4607999920845032
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,3584,0.2519040107727051
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,3072,0.22323200106620789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,8192,0.5191680192947388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,2560,0.19251200556755066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,512,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,10240,0.6399999856948853
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,256,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,2048,0.16486400365829468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,1024,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,128,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,1536,0.13619199395179749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,12288,0.7587839961051941
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,16384,0.31539198756217957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,5120,0.11980800330638885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,7168,0.16179199516773224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,8192,0.16793599724769592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,10240,0.22118400037288666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,12288,0.2734079957008362
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,65536,16384,0.9963520169258118
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,4096,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,3584,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,3072,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,1024,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,2048,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,1536,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,2560,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,256,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,512,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,128,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,7168,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,5120,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,8192,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,12288,0.158720001578331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,3584,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,10240,0.13414399325847626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,3072,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,4096,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,2560,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,16384,0.20787200331687927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,2048,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,512,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,1024,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,256,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,1536,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,128,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,12288,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,16384,0.20582400262355804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,8192,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,7168,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,10240,0.13414399325847626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,5120,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,4096,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,3072,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,3584,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,2560,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,2048,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,1536,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,512,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,8192,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,12288,0.15769599378108978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,10240,0.13414399325847626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,16384,0.20582400262355804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,5120,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,4096,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,3584,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,7168,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,12288,65536,0.7864320278167725
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,3072,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,2048,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,2560,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,1536,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,1024,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,16384,65536,1.4510079622268677
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,10240,65536,0.7874559760093689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,8192,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,10240,0.13414399325847626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,7168,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,12288,0.15667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,4096,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,16384,0.159743994474411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,5120,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,3584,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,1536,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,3072,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,2560,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,2048,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,1024,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,256,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,12288,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,7168,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,10240,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,16384,0.12902399897575378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,8192,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,8192,65536,0.779263973236084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,3584,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,3072,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,2048,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,4096,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,5120,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,1024,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,7168,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,8192,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,12288,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,10240,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,16384,0.13926400244235992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,5120,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,5120,65536,0.4833280146121979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,4096,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,3584,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,3072,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,7168,65536,0.7813119888305664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,12288,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,7168,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,8192,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,5120,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,4096,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,10240,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,16384,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,3584,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,3072,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,4096,65536,0.35123199224472046
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,8192,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,12288,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,7168,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,10240,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,5120,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,3584,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,4096,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,16384,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3584,65536,0.32051199674606323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,12288,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,8192,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,10240,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,7168,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,16384,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,4096,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,3584,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,3072,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,3072,65536,0.33689600229263306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,10240,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,12288,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,7168,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,4096,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,5120,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,16384,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2560,65536,0.32153600454330444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,16384,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,2048,65536,0.1884160041809082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,16384,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1536,65536,0.15564799308776855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,8192,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,12288,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,2048,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,1024,65536,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,12288,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,16384,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,512,65536,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,16384,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,5120,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,256,65536,0.14950400590896606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,160,128,65536,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,5120,0.26521599292755127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,4096,0.21401600539684296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,7168,0.3563520014286041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,8192,0.38809600472450256
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,2560,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,10240,0.4986880123615265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,3072,0.17612800002098083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,3584,0.19660800695419312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,2048,0.12083200365304947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,1536,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,512,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,256,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,12288,0.5826560258865356
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,1024,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,128,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,7168,0.11366400122642517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,12288,0.16896000504493713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,5120,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,10240,0.14745600521564484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,8192,0.11366400122642517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,65536,16384,0.749567985534668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,16384,0.23244799673557281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,3584,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,2560,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,4096,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,3072,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,2048,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,1536,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,512,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,1024,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,256,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,5120,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,7168,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,8192,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,12288,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,10240,0.11468800157308578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,4096,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,16384,0.16998399794101715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,2048,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,3072,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,3584,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,2560,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,1024,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,1536,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,7168,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,12288,0.12697599828243256
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,8192,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,10240,0.10444799810647964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,16384,0.15564799308776855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,3072,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,5120,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,3584,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,4096,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,2560,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,2048,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,1024,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,1536,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,16384,65536,0.7956479787826538
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,12288,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,7168,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,8192,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,10240,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,12288,65536,0.6103039979934692
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,16384,0.14745600521564484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,3072,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,4096,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,3584,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,5120,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,2560,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,2048,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,1536,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,1024,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,10240,65536,0.5601279735565186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,8192,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,12288,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,10240,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,7168,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,16384,0.130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,4096,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,5120,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,3072,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,3584,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,1536,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,12288,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,10240,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,8192,65536,0.5130239725112915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,8192,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,7168,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,16384,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,5120,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,4096,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,3584,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,3072,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,512,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,7168,65536,0.47206398844718933
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,12288,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,7168,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,16384,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,8192,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,10240,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,5120,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,4096,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,5120,65536,0.3123199939727783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,3584,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,3072,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,8192,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,12288,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,7168,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,5120,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,10240,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,16384,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,4096,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,4096,65536,0.2754560112953186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,12288,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,8192,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,10240,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,16384,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3584,65536,0.26316800713539124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,12288,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,7168,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,8192,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,10240,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,16384,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,3584,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,3072,65536,0.20787200331687927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,7168,0.21913599967956543
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,16384,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,2560,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2560,65536,0.17817600071430206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,2048,65536,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,12288,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,4096,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,2048,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,3584,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1536,65536,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,12288,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,7168,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,16384,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,3584,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,1024,65536,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,8192,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,2048,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,512,65536,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,8192,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,10240,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,256,65536,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,128,128,65536,0.23449599742889404
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,10240,0.4925439953804016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,4096,0.2088959962129593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,5120,0.2611199915409088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,7168,0.3532800078392029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,2048,0.11468800157308578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,8192,0.3829759955406189
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,3072,0.17203199863433838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,2560,0.14745600521564484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,3584,0.19148799777030945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,1536,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,512,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,1024,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,128,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,256,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,12288,0.5857279896736145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,16384,0.2181120067834854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,7168,0.10342399775981903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,8192,0.11366400122642517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,10240,0.14131200313568115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,12288,0.1669120043516159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,3584,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,5120,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,4096,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,65536,16384,0.749567985534668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,3072,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,2560,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,2048,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,1536,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,256,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,512,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,128,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,5120,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,7168,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,12288,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,8192,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,10240,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,16384,0.159743994474411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,3072,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,4096,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,3584,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,2560,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,2048,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,1536,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,1024,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,12288,0.11673600226640701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,7168,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,8192,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,10240,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,4096,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,5120,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,16384,0.1454080045223236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,3584,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,3072,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,1024,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,2048,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,2560,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,1536,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,512,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,12288,0.11366400122642517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,8192,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,16384,65536,0.7833600044250488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,12288,65536,0.5744640231132507
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,16384,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,10240,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,10240,65536,0.52019202709198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,5120,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,3584,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,7168,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,4096,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,3072,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,1024,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,5120,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,7168,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,8192,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,12288,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,10240,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,3584,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,4096,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,16384,0.11878400295972824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,1024,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,1536,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,8192,65536,0.47308799624443054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,8192,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,12288,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,7168,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,10240,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,16384,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,4096,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,3584,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,5120,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,3072,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,12288,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,10240,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,8192,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,7168,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,16384,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,7168,65536,0.4362240135669708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,5120,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,4096,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,3584,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,3072,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,5120,65536,0.28672000765800476
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,12288,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,5120,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,7168,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,10240,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,8192,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,16384,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,4096,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,3584,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,2048,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,4096,65536,0.2519040107727051
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,12288,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,16384,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,10240,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,12288,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3584,65536,0.24473600089550018
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,10240,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,16384,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,5120,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,3584,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,1024,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,10240,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,16384,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,3072,65536,0.1884160041809082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2560,65536,0.16383999586105347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,2048,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,7168,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,5120,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,2048,65536,0.15667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,12288,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,8192,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,4096,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,10240,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,16384,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,256,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1536,65536,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,1024,65536,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,10240,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,2560,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,512,65536,0.1525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,256,65536,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,8192,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,12288,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,5120,0.2590720057487488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,4096,0.21913599967956543
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,8192,0.3840000033378601
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,96,128,65536,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,12288,0.5826560258865356
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,7168,0.35123199224472046
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,3584,0.18943999707698822
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,3072,0.17100800573825836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,2560,0.14950400590896606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,2048,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,1536,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,512,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,128,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,256,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,1024,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,10240,0.48844799399375916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,12288,0.1658879965543747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,7168,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,8192,0.11264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,3584,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,10240,0.14950400590896606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,5120,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,3072,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,4096,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,65536,16384,0.7505919933319092
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,2560,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,512,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,2048,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,1536,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,16384,0.2099200040102005
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,256,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,5120,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,7168,0.07782399654388428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,8192,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,12288,0.11980800330638885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,10240,0.10342399775981903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,16384,0.16896000504493713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,4096,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,3584,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,3072,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,2560,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,2048,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,1024,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,1536,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,12288,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,16384,0.14233599603176117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,8192,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,7168,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,10240,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,5120,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,4096,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,2048,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,2560,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,3584,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,3072,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,1536,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,1024,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,16384,65536,0.7802879810333252
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,12288,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,8192,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,10240,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,5120,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,12288,65536,0.5447679758071899
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,7168,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,16384,0.12800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,4096,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,3072,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,3584,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,2560,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,10240,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,8192,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,12288,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,10240,65536,0.5038080215454102
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,7168,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,16384,0.11980800330638885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,4096,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,5120,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,3584,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,2560,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,1024,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,8192,65536,0.45875200629234314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,10240,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,7168,65536,0.4188160002231598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,12288,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,5120,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,7168,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,8192,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,16384,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,4096,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,3072,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,3584,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,12288,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,5120,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,7168,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,8192,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,10240,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,16384,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,4096,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,3584,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,3072,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,2048,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,8192,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,7168,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,12288,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,5120,65536,0.3256320059299469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,10240,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,16384,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,4096,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,3584,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,2048,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,4096,65536,0.25088000297546387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,12288,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3584,65536,0.23244799673557281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,10240,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,16384,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,12288,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,16384,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,8192,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,10240,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,3072,65536,0.18636800348758698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2560,65536,0.159743994474411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,16384,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,5120,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,16384,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,2048,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,10240,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,7168,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1536,65536,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,2048,65536,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,128,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,2048,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,16384,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,4096,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,1024,65536,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,16384,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,512,65536,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,10240,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,12288,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,8192,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,3584,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,1536,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,2048,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,256,65536,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,80,128,65536,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,4096,0.17203199863433838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,5120,0.22323200106620789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,7168,0.29183998703956604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,8192,0.32972800731658936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,10240,0.41574400663375854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,3584,0.16076800227165222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,3072,0.1454080045223236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,2048,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,2560,0.12083200365304947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,12288,0.4915199875831604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,1536,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,128,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,512,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,256,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,1024,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,65536,16384,0.6533120274543762
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,5120,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,12288,0.13926400244235992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,7168,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,8192,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,10240,0.11366400122642517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,4096,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,3584,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,16384,0.1740799993276596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,3072,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,1536,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,1024,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,2048,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,2560,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,5120,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,8192,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,7168,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,12288,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,10240,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,16384,0.12800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,4096,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,3584,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,3072,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,1536,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,2560,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,2048,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,1024,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,7168,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,8192,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,12288,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,10240,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,16384,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,5120,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,4096,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,3072,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,1024,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,2560,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,16384,65536,0.6461439728736877
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,12288,65536,0.48230400681495667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,12288,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,10240,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,8192,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,5120,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,7168,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,16384,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,4096,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,3584,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,10240,65536,0.3993600010871887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,12288,0.07782399654388428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,8192,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,7168,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,10240,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,16384,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,5120,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,3584,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,4096,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,2560,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,2048,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,8192,65536,0.3307519853115082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,7168,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,12288,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,5120,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,8192,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,10240,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,16384,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,4096,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,2048,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,128,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,7168,65536,0.3020800054073334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,8192,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,12288,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,10240,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,7168,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,16384,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,5120,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,5120,65536,0.21094399690628052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,5120,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,8192,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,12288,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,10240,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,16384,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,4096,65536,0.18534399569034576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,12288,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,5120,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,10240,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,16384,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3584,65536,0.16486400365829468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,12288,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,5120,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,16384,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,2048,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,3072,65536,0.158720001578331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,12288,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,5120,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,16384,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,4096,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2560,65536,0.15462400019168854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,8192,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,7168,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,16384,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,3584,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,4096,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,2048,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,2048,65536,0.1525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,128,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,5120,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,16384,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,4096,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,2048,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1536,65536,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,12288,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,7168,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,8192,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,3584,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,2048,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,3072,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,1024,65536,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,12288,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,512,65536,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,8192,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,10240,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,256,65536,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,3584,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,64,128,65536,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,4096,0.16998399794101715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,5120,0.21299199759960175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,7168,0.2836480140686035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,8192,0.32051199674606323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,3072,0.1382399946451187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,3584,0.15564799308776855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,10240,0.4055039882659912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,2560,0.12595200538635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,2048,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,128,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,256,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,512,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,1024,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,12288,0.4792320132255554
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,1536,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,7168,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,5120,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,65536,16384,0.6686720252037048
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,12288,0.131071999669075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,8192,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,10240,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,4096,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,16384,0.17817600071430206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,2560,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,2048,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,3072,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,3584,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,1536,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,1024,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,5120,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,7168,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,8192,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,12288,0.10035199671983719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,10240,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,2048,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,16384,0.12492799758911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,4096,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,3072,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,3584,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,1536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,1024,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,7168,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,8192,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,12288,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,10240,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,16384,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,4096,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,3072,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,3584,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,5120,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,16384,65536,0.638975977897644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,1024,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,12288,65536,0.47308799624443054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,12288,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,8192,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,7168,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,10240,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,16384,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,5120,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,3072,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,4096,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,3584,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,2048,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,10240,65536,0.3973119854927063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,12288,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,10240,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,8192,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,7168,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,16384,0.09216000139713287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,4096,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,5120,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,3584,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,3072,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,8192,65536,0.32972800731658936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,8192,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,10240,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,5120,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,7168,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,12288,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,16384,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,4096,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,7168,65536,0.30105599761009216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,12288,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,10240,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,16384,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,4096,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,3584,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,5120,65536,0.2099200040102005
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,10240,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,12288,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,8192,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,16384,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,4096,65536,0.1812479943037033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,12288,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,10240,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3584,65536,0.16383999586105347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,16384,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,5120,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,5120,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,16384,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,1536,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,3072,65536,0.16486400365829468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,7168,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,2048,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2560,65536,0.1525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,16384,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,4096,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,3584,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,2048,65536,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,5120,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1536,65536,0.14950400590896606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,5120,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,12288,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,16384,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,1024,65536,0.16998399794101715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,12288,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,512,65536,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,16384,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,3584,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,7168,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,16384,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,4096,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,3584,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,2048,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,256,65536,0.14847999811172485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,48,128,65536,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,4096,0.16383999586105347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,5120,0.2099200040102005
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,7168,0.27852800488471985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,8192,0.32153600454330444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,3584,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,3072,0.13619199395179749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,2560,0.12083200365304947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,10240,0.39423999190330505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,2048,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,512,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,1536,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,128,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,12288,0.46489599347114563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,256,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,1024,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,5120,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,7168,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,12288,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,10240,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,8192,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,65536,16384,0.6440960168838501
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,16384,0.16896000504493713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,4096,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,2560,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,3072,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,3584,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,2048,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,1536,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,1024,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,8192,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,5120,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,7168,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,12288,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,10240,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,4096,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,3584,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,16384,0.12595200538635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,3072,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,2560,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,8192,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,7168,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,12288,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,10240,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,16384,0.11878400295972824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,16384,65536,0.6399999856948853
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,3584,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,4096,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,5120,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,2560,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,7168,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,8192,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,10240,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,12288,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,12288,65536,0.47308799624443054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,16384,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,5120,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,3584,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,4096,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,512,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,256,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,10240,65536,0.3973119854927063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,12288,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,10240,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,8192,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,7168,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,16384,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,4096,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,5120,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,3584,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,3072,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,8192,65536,0.3338240087032318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,12288,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,5120,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,10240,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,8192,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,7168,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,16384,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,4096,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,7168,65536,0.3102720081806183
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,12288,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,5120,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,8192,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,10240,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,16384,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,2560,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,5120,65536,0.2027519941329956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,12288,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,8192,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,7168,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,10240,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,16384,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,5120,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,2048,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,4096,65536,0.17510400712490082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,12288,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,8192,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,7168,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,10240,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,16384,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,4096,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,2048,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3584,65536,0.16486400365829468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,5120,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,7168,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,10240,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,16384,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,3072,65536,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,8192,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,10240,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,16384,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,4096,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,3584,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2560,65536,0.1515520066022873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,8192,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,10240,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,5120,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,7168,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,3584,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,2048,65536,0.15052799880504608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,8192,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,7168,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,16384,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,4096,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,3584,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1536,65536,0.14950400590896606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,5120,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,16384,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,4096,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,1024,65536,0.14847999811172485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,3584,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,12288,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,8192,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,16384,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,512,65536,0.14847999811172485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,2560,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,12288,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,7168,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,8192,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,10240,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,4096,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,256,65536,0.14847999811172485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,32,128,65536,0.14847999811172485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,4096,0.16076800227165222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,5120,0.19865599274635315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,7168,0.2764799892902374
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,8192,0.31436800956726074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,3584,0.158720001578331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,2560,0.11161600053310394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,3072,0.12595200538635254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,10240,0.3962880074977875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,1024,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,1536,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,512,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,2048,0.09318400174379349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,12288,0.4864000082015991
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,256,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,65536,16384,0.6420480012893677
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,5120,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,12288,0.130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,10240,0.11059200018644333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,8192,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,7168,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,16384,0.17817600071430206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,3072,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,4096,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,3584,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,2560,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,2048,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,1536,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,1024,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,256,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,5120,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,8192,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,7168,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,10240,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,12288,0.09523200243711472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,2560,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,16384,0.13926400244235992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,3584,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,3072,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,2048,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,4096,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,12288,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,7168,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,8192,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,10240,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,16384,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,5120,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,3584,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,4096,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,3072,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,16384,65536,0.6461439728736877
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,12288,65536,0.4689919948577881
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,12288,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,7168,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,8192,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,10240,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,5120,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,16384,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,3072,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,3584,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,4096,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,10240,65536,0.4055039882659912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,7168,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,8192,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,12288,0.06758400052785873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,10240,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,16384,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,3072,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,5120,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,3584,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,4096,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,2048,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,8192,65536,0.3317759931087494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,5120,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,8192,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,7168,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,12288,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,10240,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,16384,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,4096,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,3584,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,7168,65536,0.3020800054073334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,12288,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,8192,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,7168,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,10240,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,16384,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,4096,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,3072,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,5120,65536,0.19968000054359436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,12288,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,7168,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,8192,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,16384,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,10240,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,3584,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,4096,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,2048,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,4096,65536,0.16998399794101715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,8192,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,12288,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,16384,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,3584,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,4096,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3584,65536,0.16076800227165222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,12288,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,5120,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,8192,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,16384,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,2048,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,3072,65536,0.15360000729560852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,5120,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,7168,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,10240,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,16384,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,4096,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2560,65536,0.14847999811172485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,12288,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,7168,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,8192,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,5120,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,10240,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,2048,65536,0.14950400590896606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,5120,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,10240,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,8192,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,7168,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,4096,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1536,65536,0.14745600521564484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,12288,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,5120,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,7168,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,8192,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,10240,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,16384,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,2560,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,1024,65536,0.14745600521564484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,12288,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,10240,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,5120,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,8192,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,7168,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,16384,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,2048,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,4096,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,512,65536,0.14643199741840363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,12288,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,5120,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,16384,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,7168,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,8192,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,10240,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,4096,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,3072,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,256,65536,0.22835199534893036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,16,128,65536,0.14643199741840363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,4096,0.16383999586105347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,5120,0.19660800695419312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,7168,0.2815999984741211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,8192,0.3235839903354645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,3584,0.14028799533843994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,3072,0.12185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,2560,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,2048,0.09011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,1536,0.0727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,10240,0.40755200386047363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,512,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,256,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,1024,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,12288,0.4792320132255554
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,65536,16384,0.6225919723510742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,7168,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,5120,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,12288,0.13312000036239624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,10240,0.1085439994931221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,8192,0.08908800035715103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,4096,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,2048,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,1536,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,2560,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,3584,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,16384,0.1802240014076233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,3072,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,1024,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,512,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,5120,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,12288,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,7168,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,8192,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,10240,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,16384,0.13516800105571747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,2560,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,4096,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,3072,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,3584,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,2048,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,7168,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,8192,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,12288,0.08396799862384796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,10240,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,16384,65536,0.6543359756469727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,16384,0.11673600226640701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,3584,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,4096,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,5120,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,2560,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,12288,65536,0.4689919948577881
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,7168,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,12288,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,8192,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,10240,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,5120,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,16384,0.09625600278377533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,4096,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,3584,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,3072,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,2560,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,10240,65536,0.40038400888442993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,12288,0.06963200122117996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,8192,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,7168,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,10240,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,5120,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,16384,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,4096,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,3584,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,3072,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,2048,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,8192,65536,0.32972800731658936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,12288,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,8192,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,10240,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,7168,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,16384,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,4096,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,5120,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,2048,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,256,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,12288,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,7168,65536,0.30105599761009216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,10240,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,16384,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,8192,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,7168,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,4096,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,5120,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,3584,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,2048,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,5120,65536,0.21196800470352173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,512,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,5120,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,7168,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,12288,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,10240,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,16384,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,3584,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,4096,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,4096,65536,0.1740799993276596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,5120,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,12288,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,8192,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,7168,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,16384,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,3072,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3584,65536,0.16281600296497345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,12288,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,8192,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,10240,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,16384,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,5120,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,4096,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,3072,65536,0.1525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,7168,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,8192,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,12288,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,10240,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,16384,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,2048,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,4096,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2560,65536,0.14847999811172485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,12288,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,7168,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,8192,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,10240,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,16384,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,4096,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,2048,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,2048,65536,0.14745600521564484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,8192,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,10240,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,5120,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,7168,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,16384,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,4096,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1536,65536,0.14745600521564484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,12288,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,7168,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,8192,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,5120,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,10240,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,4096,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,1024,65536,0.14745600521564484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,12288,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,10240,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,8192,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,7168,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,5120,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,16384,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,4096,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,3072,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,512,65536,0.14643199741840363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,12288,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,5120,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,8192,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,10240,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,7168,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,16384,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,4096,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,256,65536,0.14643199741840363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,2560,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,3584,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,3072,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,8,128,65536,0.14643199741840363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,4096,0.15564799308776855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,5120,0.20684799551963806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,7168,0.27955201268196106
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,8192,0.319487988948822
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,3584,0.13926400244235992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,2560,0.11776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,3072,0.12697599828243256
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,1536,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,10240,0.3973119854927063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,2048,0.09113600105047226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,12288,0.4679679870605469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,512,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,1024,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,256,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,128,0.07065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,65536,16384,0.605184018611908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,5120,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,7168,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,12288,0.130048006772995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,8192,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,10240,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,3584,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,4096,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,16384,0.16281600296497345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,1536,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,2048,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,2560,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,3072,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,1024,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,512,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,256,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,128,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,7168,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,8192,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,5120,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,10240,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,12288,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,16384,0.12390399724245071
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,3584,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,4096,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,3072,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,2048,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,512,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,1024,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,256,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,128,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,1536,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,7168,0.05222399905323982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,12288,0.08294399827718735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,8192,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,10240,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,16384,0.10649599879980087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,4096,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,3584,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,5120,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,1024,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,256,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,16384,65536,0.6133760213851929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,128,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,12288,65536,0.477183997631073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,10240,0.055296000093221664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,12288,0.0655359998345375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,7168,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,8192,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,16384,0.0870399996638298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,4096,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,3072,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,3584,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,5120,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,2048,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,1024,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,512,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,128,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,256,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,10240,65536,0.3840000033378601
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,8192,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,7168,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,12288,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,10240,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,16384,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,4096,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,5120,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,3584,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,8192,65536,0.3102720081806183
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,10240,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,12288,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,5120,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,16384,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,1536,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,3072,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,3584,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,512,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,7168,65536,0.2754560112953186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,12288,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,8192,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,10240,0.22118400037288666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,16384,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,5120,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,4096,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,3072,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,2048,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,5120,65536,0.2017280012369156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,5120,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,8192,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,7168,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,12288,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,10240,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,16384,0.04403200000524521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,4096,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,3072,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,1536,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,2048,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,128,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,4096,65536,0.16076800227165222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,8192,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,12288,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,7168,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,10240,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,5120,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,16384,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,4096,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,3584,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,2560,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,3072,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,2048,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,1536,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3584,65536,0.14643199741840363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,12288,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,7168,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,8192,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,10240,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,16384,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,5120,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,3072,65536,0.14336000382900238
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,4096,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,2560,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,3072,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,3584,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,2048,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,1536,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,5120,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,12288,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,7168,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,10240,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,8192,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,16384,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,3072,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,2560,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,4096,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,3584,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,1536,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,2048,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,1024,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2560,65536,0.10342399775981903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,12288,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,8192,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,10240,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,7168,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,5120,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,16384,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,3072,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,3584,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,4096,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,2048,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,2560,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,2048,65536,0.0942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,1536,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,1024,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,512,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,12288,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,10240,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,5120,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,8192,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,7168,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,16384,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1536,65536,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,2048,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,4096,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,2560,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,3584,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,3072,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,1536,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,1024,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,256,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,512,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,5120,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,12288,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,7168,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,10240,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,16384,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,8192,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,3072,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,4096,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,2560,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,3584,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,2048,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,1536,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,1024,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,256,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,512,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,128,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,1024,65536,0.04915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,12288,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,8192,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,10240,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,7168,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,512,65536,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,5120,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,16384,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,3072,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,4096,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,3584,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,2560,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,2048,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,1536,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,1024,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,512,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,256,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,12288,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,5120,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,8192,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,7168,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,16384,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,10240,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,2048,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,3584,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,2560,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,4096,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,3072,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,1536,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,256,65536,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,1024,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,256,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,512,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,128,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,4,128,65536,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,4096,0.1658879965543747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,5120,0.20582400262355804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,7168,0.26521599292755127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,8192,0.29900801181793213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,3584,0.14950400590896606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,2560,0.10956799983978271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,3072,0.12800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,1536,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,10240,0.3819519877433777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,2048,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,1024,0.05734400078654289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,512,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,256,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,12288,0.4607999920845032
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,128,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,65536,16384,0.6072319746017456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,5120,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,8192,0.08601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,7168,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,10240,0.09830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,12288,0.11264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,4096,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,3584,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,16384,0.16486400365829468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,1536,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,512,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,256,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,2560,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,3072,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,1024,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,128,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,5120,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,7168,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,12288,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,8192,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,10240,0.07680000364780426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,16384,0.13209599256515503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,3584,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,4096,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,3072,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,2560,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,2048,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,1536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,256,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,128,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,12288,0.07372800260782242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,8192,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,7168,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,10240,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,16384,0.1013759970664978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,4096,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,5120,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,3072,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,3584,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,1536,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,1024,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,128,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,16384,65536,0.609279990196228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,12288,65536,0.4546560049057007
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,12288,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,8192,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,10240,0.053247999399900436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,7168,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,5120,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,16384,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,3072,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,4096,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,2560,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,3584,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,1536,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,256,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,12288,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,10240,65536,0.38092800974845886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,10240,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,8192,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,16384,0.07475200295448303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,7168,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,5120,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,4096,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,1024,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,8192,65536,0.3123199939727783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,5120,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,12288,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,8192,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,10240,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,16384,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,2560,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,2048,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,512,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,7168,65536,0.26316800713539124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,5120,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,8192,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,10240,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,16384,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,4096,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,3072,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,3584,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,2560,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,2048,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,1536,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,256,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,5120,65536,0.1955839991569519
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,7168,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,12288,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,8192,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,5120,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,10240,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,16384,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,3584,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,2048,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,1536,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,3072,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,4096,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,2560,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,1024,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,512,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,4096,65536,0.159743994474411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,12288,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,8192,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,7168,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,10240,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,16384,0.03686400130391121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,5120,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,3072,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,3584,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,4096,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,2560,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,2048,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,1536,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,512,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3584,65536,0.13619199395179749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,8192,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,12288,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,10240,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,7168,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,5120,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,16384,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,4096,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,3584,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,2560,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,3072,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,2048,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,1024,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,512,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,1536,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,3072,65536,0.11468800157308578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,12288,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,8192,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,5120,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,10240,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,7168,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,16384,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,4096,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,3584,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,2560,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,1536,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,2048,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,3072,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2560,65536,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,1024,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,512,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,5120,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,8192,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,7168,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,12288,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,10240,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,16384,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,3584,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,3072,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,4096,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,2048,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,2560,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,1536,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,512,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,1024,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,256,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,2048,65536,0.09932799637317657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,12288,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,7168,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,8192,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,10240,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,5120,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,16384,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,2560,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,3072,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,3584,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,4096,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,1536,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,2048,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,256,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,512,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1536,65536,0.06348799914121628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,12288,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,5120,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,8192,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,7168,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,10240,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,16384,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,4096,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,3072,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,2560,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,2048,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,3584,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,1024,65536,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,1536,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,1024,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,512,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,256,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,12288,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,5120,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,8192,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,10240,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,7168,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,16384,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,2560,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,3584,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,4096,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,2048,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,3072,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,1536,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,1024,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,512,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,512,65536,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,256,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,12288,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,5120,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,8192,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,7168,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,10240,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,16384,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,4096,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,2048,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,2560,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,3072,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,3584,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,1536,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,256,65536,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,1024,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,512,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,256,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,2,128,65536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,4096,0.16383999586105347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,5120,0.20479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,7168,0.26316800713539124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,3584,0.1443839967250824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,8192,0.29900801181793213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,2560,0.10751999914646149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,3072,0.12800000607967377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,1024,0.060416001826524734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,2048,0.08806400001049042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,1536,0.06860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,10240,0.37376001477241516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,512,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,256,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,12288,0.44543999433517456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,128,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,5120,0.058368001133203506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,7168,0.0798719972372055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,8192,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,10240,0.09728000313043594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,65536,16384,0.5908480286598206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,12288,0.11468800157308578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,4096,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,16384,0.16076800227165222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,3584,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,2560,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,3072,0.04095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,1024,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,1536,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,512,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,2048,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,256,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,128,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,5120,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,7168,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,10240,0.07884799689054489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,8192,0.06451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,12288,0.08499199897050858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,16384,0.12185599654912949
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,4096,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,3584,0.048128001391887665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,3072,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,1536,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,2560,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,1024,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,512,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,128,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,7168,0.050175998359918594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,12288,0.07168000191450119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,8192,0.05632000043988228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,10240,0.06656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,16384,0.10239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,3584,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,5120,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,4096,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,3072,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,16384,65536,0.5867519974708557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,2560,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,2048,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,1024,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,256,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,128,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,12288,0.06143999844789505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,8192,0.04505600035190582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,12288,65536,0.4485119879245758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,7168,0.03993599861860275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,10240,0.05119999870657921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,16384,0.08089599758386612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,2048,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,5120,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,3584,0.026623999699950218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,4096,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,1536,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,1024,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,10240,65536,0.37171199917793274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,7168,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,12288,0.05427199974656105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,8192,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,10240,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,16384,0.07577600330114365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,5120,0.03174399957060814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,3584,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,4096,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,3072,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,2560,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,1024,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,256,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,512,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,8192,65536,0.29900801181793213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,5120,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,7168,0.02969600073993206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,8192,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,12288,0.043007999658584595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,10240,0.03788800165057182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,4096,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,16384,0.05939200147986412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,2048,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,2560,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,3072,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,3584,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,1536,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,512,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,128,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,1024,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,7168,65536,0.26419198513031006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,12288,0.035840000957250595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,7168,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,8192,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,10240,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,16384,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,5120,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,4096,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,3072,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,2560,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,3584,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,2048,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,1024,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,256,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,512,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,1536,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,128,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,5120,65536,0.19251200556755066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,12288,0.030719999223947525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,10240,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,5120,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,8192,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,7168,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,16384,0.04198399931192398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,4096,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,3584,0.02457600086927414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,1536,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,3072,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,2048,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,2560,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,4096,65536,0.15462400019168854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,1024,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,512,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,7168,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,12288,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,5120,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,10240,0.027648000046610832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,8192,0.021503999829292297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,16384,0.03481600061058998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,4096,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,3584,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,2560,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,3072,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,2048,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3584,65536,0.13619199395179749
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,1536,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,1024,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,256,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,512,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,128,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,5120,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,8192,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,7168,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,10240,0.02252800017595291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,12288,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,16384,0.03379200026392937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,2560,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,3584,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,3072,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,4096,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,2048,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,1536,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,1024,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,512,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,256,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,3072,65536,0.12083200365304947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,12288,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,8192,0.01740800030529499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,7168,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,10240,0.018432000651955605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,16384,0.025599999353289604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,5120,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,4096,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,3072,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,2560,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,3584,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,2048,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2560,65536,0.10547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,1024,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,1536,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,512,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,256,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,12288,0.03276799991726875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,7168,0.013311999849975109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,5120,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,8192,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,10240,0.015359999611973763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,16384,0.020479999482631683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,4096,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,3584,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,3072,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,1536,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,2560,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,2048,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,1024,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,512,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,256,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,2048,65536,0.08191999793052673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,12288,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,10240,0.014336000196635723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,7168,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,5120,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,8192,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,16384,0.016383999958634377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,4096,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,3584,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,3072,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1536,65536,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,2048,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,2560,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,1536,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,1024,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,128,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,256,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,512,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,12288,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,10240,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,5120,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,8192,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,7168,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,16384,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,3584,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,3072,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,2048,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,2560,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,4096,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,1024,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,512,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,1536,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,1024,65536,0.04608000069856644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,256,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,128,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,5120,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,12288,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,7168,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,8192,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,10240,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,16384,0.011264000087976456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,4096,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,3584,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,3072,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,1536,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,2560,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,2048,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,512,65536,0.028672000393271446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,1024,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,512,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,256,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,128,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,12288,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,5120,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,7168,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,8192,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,10240,0.009216000325977802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,16384,0.010239999741315842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,4096,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,3584,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,2048,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,2560,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,3072,0.01228800043463707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,256,65536,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,1536,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,1024,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,512,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,128,0.008191999979317188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,256,0.0071680000983178616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0,sq,1,128,65536,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,65536,4096,16.700586954752602
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,65536,5120,21.22605183919271
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,65536,7168,30.113006591796875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,65536,6144,25.69758707682292
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,65536,8192,33.765785725911456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,65536,10240,42.40738932291667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,65536,1536,6.796356201171875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,65536,2560,10.854399617513021
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,65536,3584,14.7381591796875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,65536,768,3.7328895568847655
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,65536,3072,13.054156494140624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,65536,2048,8.822715250651042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,65536,512,2.614476776123047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,65536,1024,4.796791585286458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,65536,128,1.5720106760660806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,65536,256,1.4896469116210938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,65536,32,0.7813461303710938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,65536,64,0.9892522811889648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,65536,12288,49.84432779947917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,16384,7168,7.353105163574218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,16384,8192,8.41308135986328
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,16384,10240,10.241058349609375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,65536,16384,65.57470703125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,16384,6144,6.239231872558594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,16384,12288,12.410606892903646
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,16384,4096,4.279227701822917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,16384,5120,5.332718912760416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,16384,2048,2.1857620239257813
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,16384,3584,3.6158121744791663
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,16384,2560,2.681378173828125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,16384,3072,3.0410069783528644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,16384,16384,16.683758544921876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,16384,1024,1.1224746704101562
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,16384,256,0.3521877288818359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,16384,512,0.5989717483520508
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,16384,768,0.8664746602376303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,16384,128,0.33809067408243815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,16384,1536,1.6443392435709636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,16384,64,0.2510848045349121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,16384,32,0.199509334564209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,12288,6144,4.816144816080729
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,12288,7168,5.591620381673177
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,12288,8192,6.368187967936198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,12288,10240,7.742941792805989
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,12288,12288,9.435647583007812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,12288,4096,3.2429056803385414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,12288,5120,4.042001088460287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,12288,3584,2.8289024353027346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,12288,3072,2.372437286376953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,12288,2048,1.6293205261230468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,12288,1024,0.8272213617960611
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,12288,1536,1.2226900736490884
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,12288,2560,2.016870371500651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,12288,768,0.6273024241129558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,12288,256,0.269380251566569
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,12288,64,0.19196586608886718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,12288,128,0.25661439895629884
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,12288,512,0.4555775960286458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,12288,32,0.15127894083658855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,12288,16384,12.27147928873698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,10240,7168,4.552942911783854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,10240,8192,5.151163736979166
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,10240,10240,6.635861206054687
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,10240,12288,7.840699768066406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,10240,5120,3.251848602294922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,10240,4096,2.675916798909505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,10240,6144,3.895978546142578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,10240,16384,10.42169189453125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,10240,3584,2.319940185546875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,10240,2048,1.3095252990722657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,10240,1024,0.7049557367960613
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,10240,1536,1.0164223988850911
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,10240,3072,1.9386027018229168
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,10240,2560,1.6600064595540363
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,10240,256,0.224017063776652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,10240,64,0.15872000058492025
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,10240,128,0.21244586308797203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,10240,512,0.3770026524861654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,10240,768,0.5312170664469401
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,10240,32,0.12581546306610109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,8192,8192,4.188501230875651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,8192,10240,5.134165445963541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,8192,12288,6.258449300130208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,8192,7168,3.665134938557943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,8192,16384,8.050039672851563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,8192,6144,3.1062698364257812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,8192,5120,2.566212209065755
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,16384,65536,70.12451171875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,8192,4096,2.0901546478271484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,8192,3584,1.7943893432617188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,8192,3072,1.5430314381917318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,8192,2560,1.3110271453857423
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,8192,1024,0.5591381072998047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,8192,2048,1.0477568308512368
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,8192,768,0.42820266087849934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,8192,1536,0.8007680257161459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,8192,512,0.3034111976623535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,8192,32,0.10318506558736165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,8192,128,0.170359468460083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,8192,64,0.12834133307139078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,8192,256,0.1783125400543213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,7168,8192,3.633732350667318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,12288,65536,51.0557861328125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,7168,10240,4.534783935546875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,7168,12288,5.52635752360026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,7168,16384,7.3698984781901045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,7168,6144,2.757222493489583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,7168,5120,2.277444203694661
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,7168,7168,3.1622825622558595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,7168,4096,1.83548583984375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,7168,1536,0.7101098378499349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,7168,2048,0.9078442891438803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,7168,3584,1.6144725799560546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,7168,2560,1.1305984497070312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,7168,3072,1.368780771891276
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,7168,512,0.2664106686909994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,7168,256,0.15800320307413737
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,7168,768,0.36915200551350913
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,7168,128,0.15155199368794758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,7168,1024,0.4823040008544922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,7168,64,0.11301546891530354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,7168,32,0.09089706738789877
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,10240,65536,43.26031494140625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,6144,8192,3.191534932454427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,6144,7168,2.8448084513346354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,6144,10240,4.0567466735839846
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,6144,12288,4.837205505371093
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,6144,4096,1.6013994852701825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,6144,6144,2.3733248392740887
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,6144,5120,2.0106581370035808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,6144,16384,6.3525548299153645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,6144,3584,1.4009685516357422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,6144,1536,0.6213631947835286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,6144,2048,0.8059562683105469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,6144,1024,0.4153002738952637
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,6144,2560,0.9810944239298502
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,6144,3072,1.1944618225097656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,6144,64,0.09775786399841309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,8192,65536,34.12667643229167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,6144,128,0.130457599957784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,6144,512,0.23084373474121095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,6144,768,0.3217407862345377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,6144,256,0.1379327932993571
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,6144,32,0.07768747011820475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,5120,7168,2.2935211181640627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,5120,8192,2.613350423177083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,5120,10240,3.2298324584960936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,5120,12288,3.962504577636719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,5120,6144,1.94334716796875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,5120,5120,1.6038570404052734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,5120,4096,1.2568575541178384
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,5120,16384,5.1303766886393225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,5120,3584,1.1197099049886068
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,5120,3072,0.95010134379069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,5120,2048,0.6505813598632812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,5120,1536,0.49991680781046555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,5120,1024,0.35566933949788415
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,5120,2560,0.8035669326782227
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,5120,768,0.2717013359069824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,5120,256,0.11424427032470703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,5120,128,0.10970453421274821
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,5120,512,0.18817706108093263
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,5120,64,0.08161280155181885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,5120,32,0.06601386864980062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,7168,65536,30.045526123046876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,4096,8192,2.115549850463867
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,4096,10240,2.6710357666015625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,4096,7168,1.8577407836914062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,4096,12288,3.2103424072265625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,4096,6144,1.5760725657145183
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,4096,4096,1.035537083943685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,4096,16384,4.316637674967447
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,4096,3584,0.9024170557657877
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,4096,5120,1.3072043100992838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,4096,1024,0.2762069384256999
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,4096,3072,0.7860906600952149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,4096,1536,0.41611948013305666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,4096,2560,0.660855484008789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,4096,2048,0.5402965545654297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,4096,512,0.1544533411661784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,4096,256,0.09308160146077474
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,4096,768,0.2186581293741862
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,4096,64,0.06734506289164224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,4096,128,0.08949759801228842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,4096,32,0.054579198360443115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,6144,65536,24.86408487955729
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3584,8192,1.886515172322591
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3584,10240,2.3955113728841146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3584,7168,1.6072704315185546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3584,12288,2.879078420003255
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3584,6144,1.3946197509765625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3584,5120,1.1573930104573569
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3584,3584,0.8178346633911133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3584,16384,3.8079147338867188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3584,4096,0.9087658564249674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3584,1024,0.23886507352193198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3584,1536,0.37515945434570314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3584,3072,0.7046143849690755
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3584,2048,0.4850005467732747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3584,2560,0.5928960164388021
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3584,768,0.1911125342051188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3584,256,0.0831829309463501
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3584,512,0.13824000358581542
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3584,64,0.06055253346761068
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3584,128,0.08011093139648437
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3584,32,0.048503466447194415
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,5120,65536,21.558614095052086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3072,8192,1.6120831807454425
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3072,10240,2.0946603139241535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3072,7168,1.4469120025634765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3072,12288,2.522521718343099
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3072,4096,0.8263338724772135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3072,6144,1.206476847330729
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3072,5120,1.0252288182576499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3072,3584,0.7227733612060547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3072,16384,3.2567977905273438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3072,1024,0.2141866683959961
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3072,3072,0.6276095708211262
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3072,1536,0.3312639872233073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3072,2048,0.429533863067627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3072,2560,0.5221034685770671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3072,768,0.16786773999532062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3072,128,0.06970026493072509
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3072,256,0.07277226448059082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3072,64,0.052121599515279136
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3072,512,0.12001279989878337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3072,32,0.04198400179545085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,4096,65536,16.4242431640625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2560,8192,1.2421461741129556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2560,7168,1.0731178283691407
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2560,10240,1.5802708943684896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2560,12288,1.850231424967448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2560,4096,0.6227285385131835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2560,5120,0.7653034845987956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2560,6144,0.9211903889973959
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2560,3584,0.5386581420898438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2560,16384,2.5801727294921877
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2560,3072,0.4751018524169922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2560,1024,0.18244266510009766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2560,1536,0.25180160204569496
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2560,2048,0.32614399592081705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2560,2560,0.3964586575826009
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2560,512,0.09905493259429932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2560,128,0.05580799976984659
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2560,768,0.14168747266133624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2560,64,0.04348586797714234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2560,256,0.06031359831492106
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2560,32,0.03502080043156942
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3584,65536,14.686173502604166
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2048,8192,1.0074111938476562
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2048,7168,0.8790698369344077
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2048,10240,1.2428287506103515
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2048,12288,1.5227904001871744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2048,4096,0.5077333450317383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2048,5120,0.6313301086425781
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2048,6144,0.7579989115397135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2048,16384,2.1142186482747394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2048,3584,0.4511402765909831
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2048,1536,0.21039786338806152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2048,1024,0.1477290630340576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2048,2048,0.27037013371785484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2048,2560,0.33058134714762366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2048,3072,0.3886421203613281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2048,512,0.08744959831237793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2048,768,0.11775999863942463
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2048,128,0.04614826838175456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2048,64,0.03642026583353679
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2048,256,0.050619733333587644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2048,32,0.02935466567675273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,3072,65536,12.977596028645834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1536,8192,0.7960917154947917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1536,7168,0.7051946640014648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1536,10240,1.0038613637288412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1536,12288,1.1783167521158853
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1536,4096,0.4101461410522461
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1536,5120,0.5084159851074219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1536,6144,0.6011221567789714
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1536,16384,1.5685972849527994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1536,3584,0.3590485254923502
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1536,2560,0.2653525352478027
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1536,2048,0.21531306902567543
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1536,1024,0.1182037353515625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1536,1536,0.16834559440612792
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1536,3072,0.3135146776835124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1536,768,0.09454933007558187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1536,128,0.036761601765950516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1536,256,0.04078933397928874
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1536,512,0.07000746726989746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1536,64,0.029047467311223346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1536,32,0.023483733336130776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2560,65536,10.832077026367188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1024,8192,0.5341183980305989
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1024,10240,0.65392640431722
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1024,7168,0.47544320424397785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1024,12288,0.770525868733724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1024,6144,0.45397332509358723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1024,4096,0.3083264032999674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1024,16384,1.015500831604004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1024,5120,0.3802111943562826
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1024,3584,0.27016534805297854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1024,1024,0.09000960191090902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1024,1536,0.12598613103230794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1024,2560,0.1988266626993815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1024,2048,0.16151893933614095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1024,3072,0.2353834629058838
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1024,512,0.052121599515279136
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1024,768,0.07069013118743897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1024,128,0.02744320034980774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1024,64,0.02177706758181254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1024,256,0.0335530678431193
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1024,32,0.0180565337340037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,2048,65536,8.494011942545573
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,768,8192,0.4011690775553386
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,768,10240,0.49879039128621416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,768,12288,0.5963434855143229
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,768,7168,0.35235840479532876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,768,6144,0.3036160151163737
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,768,16384,0.8031232198079428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,768,5120,0.25535146395365393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,768,4096,0.20684800148010254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,768,3584,0.18230613072713214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,768,1024,0.06137173175811768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,768,2048,0.10953386624654134
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,768,1536,0.0859818696975708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,768,2560,0.133188263575236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,768,3072,0.15790079434712728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,768,512,0.03553280035654704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,768,128,0.019285333156585694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,768,256,0.023449599742889404
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,768,768,0.04782079855600993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,768,64,0.015598932902018229
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,768,32,0.012936533490816752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1536,65536,6.408943176269531
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,512,8192,0.3212287902832031
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,512,7168,0.2894165356953939
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,512,10240,0.36113065083821616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,512,12288,0.41379839579264327
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,512,4096,0.20486826896667482
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,512,5120,0.25402026176452636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,512,6144,0.24722773234049478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,512,16384,0.5274282773335774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,512,3584,0.18046293258666993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,512,1536,0.08454826672871908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,512,2560,0.1372159957885742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,512,2048,0.10786133607228596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,512,1024,0.0602453351020813
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,512,3072,0.156330664952596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,512,768,0.04734293222427368
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,512,128,0.01884160041809082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,512,512,0.034986666838328045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,512,256,0.02290346622467041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,512,64,0.015155200163523355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,512,32,0.01256106694539388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,1024,65536,4.230075836181641
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,256,8192,0.15520426432291667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,256,7168,0.1405610720316569
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,256,10240,0.18875734011332196
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,256,12288,0.21978453000386558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,256,5120,0.10680320262908935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,256,4096,0.09147733052571615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,256,6144,0.12195839881896972
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,256,3584,0.08284160296122232
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,256,16384,0.28671998977661134
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,256,2048,0.05536426703135172
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,256,2560,0.06717440287272135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,256,1024,0.030822400252024335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,256,3072,0.07929173310597738
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,256,1536,0.042905600865681966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,256,256,0.012117333213488261
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,256,768,0.024473599592844644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,256,512,0.018227199713389076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,256,128,0.010478933652242024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,256,64,0.008567466338475545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,256,32,0.007099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,768,65536,3.1227561950683596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,128,7168,0.10605226357777912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,128,8192,0.10830506483713787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,128,10240,0.12625919977823893
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,128,12288,0.14554452896118164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,128,4096,0.059528533617655435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,128,5120,0.07621973355611165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,128,6144,0.08642559846242269
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,128,16384,0.19128319422403972
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,128,3584,0.05638826688130697
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,128,1536,0.024098134040832518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,128,1024,0.01791999936103821
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,128,2048,0.03348480065663655
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,128,2560,0.04171093304951985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,128,3072,0.048469332853953044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,128,512,0.011844266454378765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,128,256,0.00993280013402303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,128,128,0.0075434664885203045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,128,768,0.014916266997655234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,128,64,0.005905066430568695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,128,32,0.005256533126036326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,512,65536,2.059639485677083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,64,7168,0.09878186384836832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,64,8192,0.10103466510772705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,64,10240,0.12069546381632487
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,64,12288,0.1386496067047119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,64,5120,0.0579584002494812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,64,4096,0.04679679870605469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,64,6144,0.06905173460642497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,64,16384,0.18647039731343587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,64,3584,0.043724799156188966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,64,1024,0.011502933502197266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,64,2560,0.03232426643371582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,64,2048,0.024200532833735147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,64,3072,0.038365864753723146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,64,1536,0.014882133404413859
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,64,256,0.007816533247629803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,64,128,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,64,64,0.005529599885145823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,64,512,0.008328533172607422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,64,768,0.010137599706649781
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,64,32,0.005154133339722951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,256,65536,1.052501360575358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,32,8192,0.08929279645284018
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,32,7168,0.08069120248158773
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,32,10240,0.11042133172353108
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,32,12288,0.1304917335510254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,32,4096,0.045841066042582194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,32,6144,0.06655999819437662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,32,5120,0.055978667736053464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,32,3584,0.04116479953130086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,32,16384,0.175820795694987
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,32,2048,0.02174293398857117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,32,1024,0.011195733149846395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,32,1536,0.014574933052062988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,32,3072,0.037546666463216145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,32,2560,0.03222186764081319
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,32,256,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,32,128,0.004949333270390829
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,32,64,0.003857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,32,512,0.008055466910203297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,32,768,0.009693866968154908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,128,65536,0.7020544052124024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,32,32,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,65536,7168,14.678801472981771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,65536,6144,12.641075642903646
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,65536,8192,16.825924682617188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,65536,10240,21.143961588541664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,64,65536,0.6792874654134115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,65536,3584,7.509572347005208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,65536,5120,10.609220377604167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,65536,4096,8.555349731445313
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,65536,3072,6.534314473470052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,65536,2560,5.334767150878906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,65536,1536,3.4072575887044274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,65536,12288,24.546270751953124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,65536,768,1.8174293518066407
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,65536,1024,2.363391876220703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,65536,256,0.7209301630655924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,65536,2048,4.395076497395833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,65536,512,1.2337151845296224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,65536,128,0.6988800048828125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,65536,64,0.49995091756184895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,65536,32,0.39321600596110023
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,65536,16384,33.90945231119792
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,16384,8192,4.225297037760416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8192,32,65536,0.6615039825439453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,16384,7168,3.700735982259115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,16384,10240,5.086037190755208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,16384,12288,6.125465393066406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,16384,5120,2.6218495686848957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,16384,4096,2.0605269114176434
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,16384,3584,1.8320383707682293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,16384,6144,3.192729695638021
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,16384,16384,8.29310302734375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,16384,3072,1.518762715657552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,16384,512,0.30839465459187826
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,16384,256,0.18083839416503905
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,16384,1024,0.5643264134724935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,16384,768,0.4341077486673991
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,16384,1536,0.797320556640625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,16384,2560,1.281979751586914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,16384,128,0.17363626162211102
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,16384,64,0.1295360008875529
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,16384,32,0.10287786324818929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,16384,2048,1.042841593424479
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,12288,6144,2.4405675252278645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,12288,7168,2.833817545572917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,12288,8192,3.1683583577473957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,12288,10240,4.001177724202474
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,12288,12288,4.782250467936198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,12288,4096,1.6074410756429036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,12288,3584,1.359837849934896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,12288,3072,1.1917994181315104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,12288,5120,2.022161102294922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,12288,16384,6.348048909505208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,12288,1024,0.43769172032674153
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,12288,1536,0.6191445032755534
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,12288,2560,0.9817770640055338
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,12288,2048,0.8053759892781576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,12288,768,0.33286825815836585
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,12288,256,0.13851307233174642
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,12288,512,0.2309802691141764
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,12288,128,0.13267626762390136
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,12288,64,0.09854293664296468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,12288,32,0.07799466451009116
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,10240,7168,2.2811648050944013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,10240,8192,2.548394775390625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,10240,10240,3.189111582438151
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,10240,12288,3.8906539916992187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,10240,16384,5.193864440917968
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,10240,5120,1.5855616251627604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,10240,4096,1.241053899129232
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,10240,6144,1.9469994862874347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,10240,3584,1.0981717427571616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,10240,1024,0.35611305236816404
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,10240,3072,0.9496917088826498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,10240,1536,0.5035008112589519
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,10240,2048,0.6421503702799479
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,10240,2560,0.7928831736246745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,10240,512,0.1946282704671224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,10240,256,0.1164629300435384
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,10240,128,0.11233279705047608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,10240,768,0.2712575912475586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,10240,64,0.08219306468963623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,10240,32,0.06621866623560588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,16384,65536,33.750732421875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,8192,8192,2.1158911387125654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,8192,10240,2.623931630452474
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,8192,12288,3.1782569885253906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,8192,6144,1.573956298828125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,8192,7168,1.8476715087890625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,8192,16384,4.195054880777994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,8192,5120,1.2839253743489585
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,8192,4096,1.0222250620524087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,8192,3072,0.7835306803385417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,8192,2560,0.6539946873982747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,8192,3584,0.9131349563598633
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,8192,1536,0.4165290514628093
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,8192,2048,0.534220822652181
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,8192,1024,0.2956629435221354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,8192,512,0.156876802444458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,8192,768,0.21797547340393067
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,8192,128,0.09069226582845052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,8192,256,0.09352533022562662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,8192,64,0.06816426912943521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,8192,32,0.054647465546925865
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,12288,65536,25.576481119791666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,7168,8192,1.8796885172526043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,7168,10240,2.371379089355469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,7168,7168,1.5801344553629557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,7168,12288,2.84955317179362
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,7168,16384,3.750673166910807
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,7168,6144,1.393561553955078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,7168,5120,1.15599365234375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,7168,4096,0.9319082895914713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,7168,3584,0.8058197021484375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,10240,65536,21.372688802083335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,7168,2048,0.48346452713012694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,7168,1536,0.37710507710774743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,7168,1024,0.2642602602640788
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,7168,768,0.197597869237264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,7168,2560,0.5935445149739583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,7168,3072,0.6990847905476888
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,7168,512,0.13827412923177082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,7168,32,0.04836693207422892
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,7168,256,0.08335359891255698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,7168,128,0.08130559921264649
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,7168,64,0.06038186550140381
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,6144,7168,1.4386175791422526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,6144,8192,1.6261461893717448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,6144,10240,2.0852736155192058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,6144,12288,2.5105748494466145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,6144,16384,3.3177258809407553
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,6144,6144,1.2345685323079427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,6144,5120,1.012224006652832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,6144,4096,0.8184490839640299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,6144,3584,0.7209642410278321
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,8192,65536,17.061512247721353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,6144,1536,0.3318101247151693
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,6144,2048,0.4261205355326335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,6144,3072,0.6249472300211589
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,6144,2560,0.5286229451497395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,6144,1024,0.2340522607167562
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,6144,128,0.06997333367665609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,6144,256,0.07342080275217691
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,6144,512,0.1203541358311971
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,6144,768,0.17192959785461426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,6144,64,0.052121599515279136
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,6144,32,0.04167679945627849
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,5120,7168,1.0791253407796224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,5120,8192,1.249791971842448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,5120,10240,1.5797247568766275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,5120,12288,1.9063125610351563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,5120,5120,0.774553616841634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,5120,6144,0.919483757019043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,5120,16384,2.53658447265625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,5120,4096,0.6246400197347005
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,5120,3584,0.545689582824707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,5120,3072,0.47472639083862306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,5120,2560,0.39976959228515624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,5120,1536,0.25384960174560545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,5120,2048,0.3251200040181478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,5120,1024,0.1777664025624593
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,7168,65536,14.389486694335938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,5120,256,0.06085973183314005
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,5120,128,0.0566271980603536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,5120,512,0.09949866930643717
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,5120,64,0.043281066417694095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,5120,768,0.14175573984781903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,5120,32,0.03519146839777629
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,4096,7168,0.8665770848592123
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,4096,8192,1.0235221227010092
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,4096,10240,1.283959452311198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,4096,12288,1.549619166056315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,4096,5120,0.6397610982259114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,4096,4096,0.5140821456909179
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,4096,6144,0.7466325124104818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,4096,16384,2.0680362701416017
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,4096,3584,0.4546901384989421
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,4096,1536,0.20985172589619955
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,4096,2048,0.2693120002746582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,4096,2560,0.3316735903422038
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,4096,3072,0.394376532236735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,6144,65536,12.552840169270834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,4096,1024,0.1481386661529541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,4096,512,0.08151040077209473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,4096,768,0.11816960175832111
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,4096,256,0.05102933247884115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,4096,64,0.03669333457946777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,4096,128,0.04648960034052531
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,4096,32,0.02945706645647685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3584,7168,0.8773973464965821
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3584,8192,1.003656514485677
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3584,10240,1.2346368153889975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3584,12288,1.492650604248047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3584,4096,0.5080405235290527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3584,16384,2.0210346221923827
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3584,5120,0.6365184148152669
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3584,6144,0.753766377766927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3584,3584,0.4501845359802246
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,5120,65536,10.686737060546875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3584,1024,0.1305941343307495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3584,768,0.10448213418324788
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3584,1536,0.2081450621287028
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3584,2560,0.3284991900126139
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3584,2048,0.2691754659016927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3584,3072,0.38806187311808266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3584,256,0.04631893237431844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3584,512,0.07424000104268393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3584,64,0.03587413231531779
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3584,128,0.04495360056559245
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3584,32,0.02757973273595174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3072,7168,0.7067306518554688
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3072,8192,0.8018261591593424
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3072,10240,0.9930410385131836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3072,12288,1.2054186503092448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3072,16384,1.6274431864420573
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3072,5120,0.5087914784749349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3072,6144,0.608460807800293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3072,4096,0.41000960667928055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3072,3584,0.36249599456787107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,4096,65536,8.62194315592448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3072,1536,0.1679360071818034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3072,2560,0.26504534085591636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3072,1024,0.11871573130289714
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3072,3072,0.31447038650512693
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3072,2048,0.21681493123372397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3072,768,0.09478826522827148
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3072,128,0.03662506739298503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3072,512,0.06980266571044921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3072,64,0.029047467311223346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3072,256,0.04078933397928874
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3072,32,0.023415466149648033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2560,8192,0.6056277592976888
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2560,7168,0.5339818954467773
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2560,10240,0.7575210571289063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2560,12288,0.9077418645222982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2560,4096,0.31201279958089195
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2560,6144,0.4583423932393392
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2560,16384,1.2224170684814453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2560,5120,0.38594560623168944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2560,3584,0.2727936108907064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3584,65536,7.23063456217448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2560,3072,0.23784106572469077
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2560,2048,0.16435200373331707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2560,2560,0.20107946395874024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2560,1024,0.09151146411895753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2560,1536,0.12799999713897706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2560,768,0.0722261349360148
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2560,128,0.028501333793004353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2560,256,0.034918399651845296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2560,512,0.053623465696970615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2560,64,0.022801067431767783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2560,32,0.018705066045125326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2048,7168,0.5325141270955404
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2048,8192,0.5318997383117676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2048,10240,0.655291748046875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2048,12288,0.7792298634847005
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2048,4096,0.3078826586405436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2048,5120,0.37833385467529296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2048,16384,1.0377898534138998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2048,6144,0.45455360412597656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,3072,65536,6.331187438964844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2048,3584,0.27108694712320963
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2048,2560,0.19783679644266766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2048,3072,0.23541760444641113
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2048,1024,0.08983893394470215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2048,768,0.07062186400095621
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2048,1536,0.12929706573486327
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2048,2048,0.16517119407653807
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2048,512,0.05198506514231364
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2048,256,0.03372373183568318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2048,64,0.021845332781473794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2048,128,0.027716267108917236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2048,32,0.0180565337340037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1536,7168,0.35225601196289064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1536,8192,0.397926394144694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1536,10240,0.496674124399821
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1536,12288,0.5945685068766277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1536,5120,0.25340587298075357
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1536,4096,0.20602879524230958
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1536,16384,0.8121344248453776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1536,6144,0.303547732035319
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1536,3584,0.18251093228658039
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2560,65536,5.261824035644532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1536,1024,0.06154239972432455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1536,1536,0.08564053376515707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1536,2048,0.10994346936543782
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1536,2560,0.13373440106709797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1536,3072,0.1572864055633545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1536,768,0.04778666496276855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1536,512,0.03546453317006429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1536,128,0.019319466749827065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1536,64,0.015633066495259605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1536,256,0.0233130673567454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1536,32,0.012936533490816752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1024,7168,0.2821802775065104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1024,8192,0.32119468053181965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1024,10240,0.3531434694925944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1024,12288,0.4124330520629883
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1024,16384,0.5490346908569336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1024,6144,0.2453162670135498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1024,5120,0.2532351970672607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1024,4096,0.20435627301534018
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1024,3584,0.18094080289204914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,2048,65536,4.184814961751302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1024,2560,0.13216426372528076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1024,1536,0.08454826672871908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1024,3072,0.1567402680714925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1024,2048,0.10816853046417237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1024,1024,0.06014293432235718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1024,768,0.04720640182495117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1024,128,0.018875734011332194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1024,512,0.03495253324508667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1024,256,0.02290346622467041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1024,32,0.012526933352152506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1024,64,0.014984533190727234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,768,7168,0.17547945976257323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,768,8192,0.200601593653361
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,768,10240,0.24791040420532226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,768,12288,0.2951509475708008
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,768,5120,0.1282047986984253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,768,16384,0.40946346918741866
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,768,4096,0.10379947026570638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,768,6144,0.15155199368794758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,768,3584,0.09164799849192301
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,768,2048,0.05573973258336386
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,768,2560,0.06809600194295248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,768,3072,0.08004266421000163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1536,65536,3.157299296061198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,768,1024,0.03160746693611145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,768,1536,0.04369066556294759
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,768,64,0.008840533097585042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,768,256,0.01269760032494863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,768,512,0.018705066045125326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,768,768,0.025565866629282636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,768,32,0.0075434664885203045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,768,128,0.010786133011182149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,512,7168,0.1386837323506673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,512,8192,0.1544874668121338
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,512,10240,0.18722133636474608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,512,12288,0.21841920216878258
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,512,16384,0.284228261311849
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,1024,65536,2.0876628875732424
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,512,4096,0.09062399864196777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,512,5120,0.1060863971710205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,512,3584,0.08246613343556722
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,512,6144,0.12137813568115234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,512,1024,0.030344533920288085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,512,2048,0.0549888014793396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,512,2560,0.0669696013132731
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,512,1536,0.042973868052164715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,512,3072,0.07751680215199788
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,512,768,0.024405332406361897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,512,256,0.012151466806729634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,512,512,0.018261333306630455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,512,128,0.010308266679445902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,512,32,0.007202133536338806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,512,64,0.008601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,256,7168,0.0808618704477946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,256,8192,0.0876202662785848
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,256,10240,0.10362880229949951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,256,12288,0.11950080394744873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,256,5120,0.06737919648488364
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,768,65536,1.5084885915120443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,256,16384,0.15322453180948895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,256,4096,0.05471573273340861
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,256,6144,0.07260159651438394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,256,3584,0.04922026793162028
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,256,1536,0.024064000447591147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,256,768,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,256,1024,0.01791999936103821
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,256,2048,0.03020799954732259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,256,2560,0.03635199864705403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,256,3072,0.042632532119750974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,256,64,0.005836800237496694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,256,256,0.008942932883898417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,256,32,0.0053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,256,512,0.01225386659304301
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,256,128,0.00747519979874293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,128,8192,0.06420480012893677
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,128,7168,0.057275732358296715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,128,10240,0.07683413028717041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,128,12288,0.0897706667582194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,128,5120,0.04666026830673218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,128,16384,0.11223039627075196
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,128,4096,0.036761601765950516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,128,6144,0.05191680192947388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,512,65536,0.9834154764811197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,128,3584,0.031061333417892457
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,128,1024,0.010990933577219645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,128,1536,0.014404267072677612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,128,768,0.009727999567985535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,128,2048,0.017373865842819212
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,128,2560,0.020855466524759926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,128,3072,0.024132267634073893
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,128,256,0.006348800162474315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,128,128,0.0052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,128,512,0.007816533247629803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,128,64,0.004710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,128,32,0.003925333420435587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,64,7168,0.051609599590301515
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,64,8192,0.05935786565144857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,64,10240,0.06823253631591797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,64,12288,0.0748202641805013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,64,5120,0.031573333342870075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,64,6144,0.045841066042582194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,64,4096,0.024644267559051514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,64,16384,0.0992256005605062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,64,3584,0.01904639999071757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,256,65536,0.5254144032796224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,64,1536,0.01088853379090627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,64,2048,0.012526933352152506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,64,3072,0.016179200013478598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,64,2560,0.014916266997655234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,64,1024,0.008533333738644917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,64,768,0.007987200220425924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,64,32,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,64,512,0.006826666494210561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,64,64,0.003925333420435587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,64,128,0.00477866679430008
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,64,256,0.005393066505591074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,32,7168,0.040516265233357746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,32,8192,0.048913065592447916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,32,10240,0.055978667736053464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,32,12288,0.06597973505655924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,128,65536,0.3725994745890299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,32,6144,0.03560106754302979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,32,5120,0.030719999472300214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,32,16384,0.09216000239054362
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,32,4096,0.02187946637471517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,32,3584,0.01648640036582947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,32,3072,0.014711466431617738
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,32,2048,0.011229866743087768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,32,1024,0.007782400151093801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,32,1536,0.009796266754468281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,32,2560,0.013038933277130127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,32,768,0.007133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,32,64,0.0034815999368826545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,32,128,0.004300799965858459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,32,256,0.005256533126036326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,32,512,0.006109866499900818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,32,32,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,64,65536,0.340445868174235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,65536,5120,5.337053934733072
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4096,32,65536,0.3273386637369792
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,65536,12288,12.410879516601563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,65536,6144,6.205405680338542
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,65536,7168,7.2536061604817705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,65536,8192,8.184251912434895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,65536,3072,3.2265556335449217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,65536,4096,4.282811482747396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,65536,3584,3.7644287109375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,65536,1536,1.6098645528157551
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,65536,10240,10.39479471842448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,65536,2048,2.1549397786458333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,65536,2560,2.6542762756347655
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,65536,1024,1.1352063496907552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,65536,768,0.8931669235229492
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,65536,512,0.6112255732218425
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,65536,256,0.3618133227030436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,65536,32,0.20008959770202636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,65536,128,0.35925334294637046
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,65536,64,0.25712639490763345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,16384,6144,1.5546709696451821
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,16384,12288,3.1833770751953123
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,16384,7168,1.8482859293619793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,16384,8192,2.077525329589844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,16384,10240,2.596795654296875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,16384,16384,4.285576375325521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,16384,4096,1.0374485015869142
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,16384,3584,0.9128618876139323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,16384,3072,0.7773183822631836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,16384,2560,0.6541994730631511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,65536,16384,16.690721638997395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,16384,2048,0.5403647740681966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,16384,5120,1.2920149485270183
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,16384,1536,0.41690454483032224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,16384,512,0.15592106183369953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,16384,64,0.06922240257263183
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,16384,128,0.0928767999013265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,16384,256,0.0958463986714681
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,16384,768,0.2203989346822103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,16384,1024,0.2956629435221354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,16384,32,0.05451093514760336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,12288,6144,1.2278101603190104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,12288,7168,1.4370816548665366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,12288,8192,1.615667215983073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,12288,12288,2.4949076334635416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,12288,10240,2.0741802215576173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,12288,5120,1.0170368194580077
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,12288,4096,0.816264533996582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,12288,3584,0.7160832087198894
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,12288,3072,0.6269269307454427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,12288,2560,0.5287935892740886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,12288,16384,3.3246548970540366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,12288,2048,0.42748587926228837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,12288,512,0.12352853616078693
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,12288,768,0.17353386878967286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,12288,1024,0.2346325397491455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,12288,1536,0.3320490519205729
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,12288,256,0.07437653541564941
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,12288,64,0.05246293147404989
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,12288,128,0.07181653181711832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,12288,32,0.04194986820220947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,10240,7168,1.08492800394694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,10240,8192,1.2395861307779947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,10240,10240,1.579144541422526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,10240,12288,1.8755242665608722
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,10240,6144,0.9255594889322916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,10240,16384,2.580036163330078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,10240,5120,0.7691946665445963
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,10240,3072,0.4747605323791504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,10240,3584,0.5428565343221028
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,10240,4096,0.6240255991617839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,10240,2048,0.32580267588297523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,10240,2560,0.39915520350138345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,10240,512,0.10062506993611653
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,10240,1536,0.2527232011159261
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,16384,65536,16.81981404622396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,10240,1024,0.1779370625813802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,10240,768,0.1421994686126709
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,10240,256,0.061610666910807285
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,10240,128,0.05935786565144857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,10240,32,0.035396265983581546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,10240,64,0.043246932824452716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,8192,7168,0.8878421147664388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,8192,8192,1.0084010442097981
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,8192,10240,1.2770304361979166
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,8192,5120,0.6378496170043946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,8192,6144,0.7622997283935546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,8192,12288,1.5483221689860025
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,8192,4096,0.5155839920043945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,8192,16384,2.094626108805339
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,8192,3584,0.4548949241638184
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,8192,3072,0.39458131790161133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,8192,2560,0.33239040374755857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,8192,2048,0.27067734400431315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,8192,1024,0.1493674596150716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,8192,1536,0.21084159215291343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,8192,768,0.11905706723531087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,8192,512,0.08905386924743652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,8192,128,0.0504149317741394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,8192,256,0.05147306521733602
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,8192,64,0.03816106716791789
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,8192,32,0.02935466567675273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,12288,65536,12.402687581380208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,7168,8192,1.0073429107666017
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,7168,7168,0.8643242518107096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,7168,12288,1.423974355061849
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,7168,10240,1.2566869099934896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,7168,16384,1.9168256123860676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,7168,6144,0.7581013361612956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,7168,4096,0.5104981422424316
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,10240,65536,10.414762369791667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,7168,3072,0.39069013595581054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,7168,5120,0.6252885182698568
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,7168,3584,0.452130126953125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,7168,2048,0.2677760124206543
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,7168,1024,0.14742186864217122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,7168,1536,0.20831573804219566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,7168,2560,0.33034238815307615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,7168,768,0.10451626777648926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,7168,512,0.07526400089263915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,7168,256,0.046830932299296066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,7168,128,0.04775253136952718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,7168,64,0.03583999872207642
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,7168,32,0.027374933163324993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,6144,7168,0.7061845143636067
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,6144,10240,1.0017109553019206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,6144,8192,0.7959893544514973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,6144,12288,1.204531224568685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,8192,65536,8.42796376546224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,6144,16384,1.6287061055501302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,6144,6144,0.608460807800293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,6144,5120,0.5087914784749349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,6144,4096,0.4079615910847981
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,6144,3584,0.3630762736002604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,6144,3072,0.31825920740763347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,6144,1536,0.16868693033854168
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,6144,2048,0.21637120246887206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,6144,768,0.09454933007558187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,6144,1024,0.11830613613128663
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,6144,2560,0.26617172559102376
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,6144,256,0.04174506664276123
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,6144,512,0.07010986804962158
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,6144,64,0.0291157325108846
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,6144,128,0.03853653271993001
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,6144,32,0.023517866929372154
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,5120,8192,0.6077098846435547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,5120,7168,0.526199467976888
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,5120,10240,0.7569407780965169
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,5120,12288,0.9007104237874349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,7168,65536,7.393075052897136
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,5120,16384,1.2172288258870443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,5120,4096,0.3110912005106608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,5120,6144,0.4598101298014323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,5120,5120,0.3809280077616374
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,5120,3584,0.27374931971232097
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,5120,2560,0.20111360549926757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,5120,3072,0.23756799697875977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,5120,2048,0.16390825907389323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,5120,1536,0.1276245355606079
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,5120,512,0.05403306484222412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,5120,1024,0.09147733052571615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,5120,768,0.07208960056304932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,5120,128,0.029013333717981975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,5120,256,0.03515733480453491
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,5120,32,0.01890986760457357
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,5120,64,0.022835199038187662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,4096,7168,0.5276330629984538
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,4096,8192,0.5337429046630859
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,4096,10240,0.6519466400146484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,4096,12288,0.7766698837280274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,6144,65536,6.212403361002604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,4096,16384,1.0242389043172202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,4096,5120,0.38099625905354817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,4096,6144,0.4547242800394694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,4096,3584,0.2715306599934896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,4096,4096,0.30702934265136717
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,4096,2048,0.16223573684692383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,4096,2560,0.198963197072347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,4096,3072,0.2338816006978353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,4096,1024,0.08901973565419516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,4096,1536,0.12619093259175618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,4096,768,0.07113386789957682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,4096,512,0.05266773303349813
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,4096,256,0.03399680058161418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,4096,64,0.02177706758181254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,4096,128,0.02764799992243449
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,4096,32,0.018261333306630455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3584,7168,0.46868480046590166
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3584,8192,0.5237418810526531
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3584,10240,0.6516735712687175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,5120,65536,5.256192016601562
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3584,12288,0.7383040110270183
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3584,16384,0.9747114817301432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3584,4096,0.30539093017578123
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3584,6144,0.4078250567118327
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3584,5120,0.37604694366455077
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3584,3072,0.23364267349243165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3584,3584,0.2706432024637858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3584,2560,0.19800747235616048
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3584,2048,0.16097280184427898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3584,1536,0.12554240226745605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3584,256,0.03368959824244182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3584,1024,0.0884394645690918
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3584,768,0.0703488032023112
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3584,512,0.051950931549072266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3584,128,0.02754559914271037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3584,32,0.017988266547520955
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3584,64,0.021845332781473794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3072,8192,0.401032543182373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3072,7168,0.35232426325480143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,4096,65536,4.209527587890625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3072,12288,0.5949098587036132
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3072,10240,0.4972544034322103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3072,16384,0.7875584284464519
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3072,5120,0.25497600237528484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3072,3072,0.15783252716064453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3072,4096,0.20548267364501954
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3072,6144,0.30351359049479165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3072,3584,0.1823402722676595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3072,768,0.048196268081665036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3072,2560,0.13410986264546712
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3072,2048,0.10936319828033447
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3072,1536,0.08543573220570883
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3072,512,0.03580586512883504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3072,1024,0.06137173175811768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3072,256,0.023347200949986775
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3072,128,0.019319466749827065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3072,64,0.015530666708946228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3072,32,0.012970667084058127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2560,7168,0.29412692387898765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2560,8192,0.32494932810465493
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2560,10240,0.3947861353556315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3584,65536,3.7981867472330726
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2560,12288,0.4701525370279948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2560,16384,0.6128981272379558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2560,3584,0.18169172604878742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2560,4096,0.2056191921234131
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2560,3072,0.15796906153361004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2560,5120,0.25415679613749187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2560,6144,0.30190931955973305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2560,2560,0.13315412998199463
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2560,1024,0.06109866698582968
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2560,1536,0.08540159861246745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2560,768,0.04792319933573405
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2560,2048,0.10898773670196533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2560,512,0.03577173153559367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2560,128,0.01925119956334432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2560,256,0.023142399390538533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2560,32,0.012936533490816752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2560,64,0.01539413332939148
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2048,7168,0.28248745600382485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2048,10240,0.35966294606526694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,3072,65536,3.1261695861816405
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2048,8192,0.32129707336425783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2048,12288,0.41509545644124346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2048,16384,0.5371903737386068
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2048,3072,0.156876802444458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2048,4096,0.20374186833699545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2048,6144,0.24657920201619468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2048,3584,0.1806335926055908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2048,5120,0.2522453308105469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2048,2560,0.1325055996576945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2048,768,0.04730879863103231
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2048,2048,0.10847573280334473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2048,512,0.03502080043156942
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2048,1024,0.05946026643117269
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2048,1536,0.08407039642333984
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2048,64,0.015052800377209982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2048,256,0.02290346622467041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2048,128,0.01884160041809082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2048,32,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1536,7168,0.17561599413553874
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2560,65536,2.6173439025878906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1536,8192,0.19985067049662272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1536,10240,0.2478421370188395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1536,12288,0.2969600041707357
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1536,16384,0.4040704091389974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1536,5120,0.12786346276601154
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1536,4096,0.1035264015197754
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1536,6144,0.15155199368794758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1536,3072,0.08014506498972575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1536,3584,0.09198933442433675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1536,2560,0.06772053241729736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1536,1024,0.031573333342870075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1536,1536,0.04369066556294759
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1536,2048,0.055637331803639736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1536,768,0.025634133815765382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1536,512,0.018705066045125326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1536,256,0.012731732924779257
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1536,128,0.0106495996316274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1536,64,0.008942932883898417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1536,32,0.007577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,2048,65536,2.1291348775227865
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1024,7168,0.1392639954884847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1024,8192,0.15458985964457195
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1024,10240,0.18701653480529784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1024,12288,0.21681493123372397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1024,16384,0.2840575853983561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1024,6144,0.1215488036473592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1024,5120,0.10574506918589274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1024,3072,0.07307946681976318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1024,4096,0.0897706667582194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1024,3584,0.08082773685455322
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1024,2560,0.06700373490651448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1024,512,0.018193066120147705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1024,1024,0.03020799954732259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1024,1536,0.04293973445892334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1024,2048,0.05495466788609823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1024,768,0.024405332406361897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1024,256,0.012151466806729634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1024,64,0.008635733524958293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1024,128,0.010308266679445902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1024,32,0.00730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1536,65536,1.5316650390625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,768,7168,0.1008298635482788
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,768,8192,0.11335679690043132
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,768,10240,0.13701119422912597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,768,12288,0.1607338587443034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,768,6144,0.08946346441904704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,768,16384,0.209169069925944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,768,4096,0.05492053429285685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,768,5120,0.07686826388041178
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,768,3584,0.04904959996541341
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,768,3072,0.04293973445892334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,768,768,0.015530666708946228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,768,2560,0.03696639935175578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,768,512,0.01225386659304301
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,768,1536,0.024644267559051514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,768,1024,0.01843199928601583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,768,2048,0.030754133065541582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,768,64,0.006212266782919565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,768,32,0.0052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,768,128,0.00795306662718455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,768,256,0.00962559978167216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,1024,65536,0.986897087097168
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,512,7168,0.08188587029774984
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,512,8192,0.08765439987182617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,512,10240,0.10506239732106526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,512,12288,0.12124160130818684
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,512,6144,0.07140693664550782
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,512,16384,0.15278080304463704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,512,4096,0.054306133588155114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,512,5120,0.06642346779505412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,512,3584,0.04833279848098755
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,512,2560,0.03638613224029541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,512,3072,0.04235946734746297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,512,1024,0.01791999936103821
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,512,1536,0.024029866854349772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,512,768,0.014882133404413859
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,512,2048,0.030139732360839843
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,512,256,0.008874666690826417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,512,512,0.012185600399971009
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,512,128,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,512,64,0.005870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,512,32,0.0052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,768,65536,0.749124272664388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,256,7168,0.048093867301940915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,256,8192,0.05300906499226889
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,256,10240,0.056149331728617344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,256,12288,0.06396586497624715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,256,16384,0.08144213358561198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,256,6144,0.04450986782709758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,256,4096,0.028808534145355225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,256,5120,0.03430399894714355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,256,3072,0.024132267634073893
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,256,3584,0.027374933163324993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,256,2560,0.020753065745035805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,256,2048,0.017407999436060587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,256,768,0.009591466188430786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,256,1024,0.010956799983978272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,256,1536,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,256,512,0.007816533247629803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,256,256,0.006109866499900818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,256,128,0.005256533126036326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,256,64,0.00481279989083608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,256,32,0.003788800040880839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,512,65536,0.5204309463500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,128,7168,0.03242666721343994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,128,8192,0.037375998497009275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,128,10240,0.04304213523864746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,128,12288,0.04771840174992879
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,128,16384,0.062463998794555664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,128,6144,0.026282666126887004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,128,4096,0.01996799906094869
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,128,5120,0.02239146629969279
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,128,3584,0.019080533583958944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,128,3072,0.015940266847610473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,128,2560,0.014779733618100485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,128,2048,0.01242453356583913
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,128,1536,0.010547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,128,1024,0.008533333738644917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,128,768,0.0076799998680750535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,128,256,0.0054954667886098225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,128,512,0.0065194666385650635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,128,64,0.003925333420435587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,128,128,0.00477866679430008
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,128,32,0.0032768001159032187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,64,7168,0.024234666426976522
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,64,8192,0.03078826665878296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,256,65536,0.3084287961324056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,64,10240,0.03614720106124878
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,64,12288,0.045841066042582194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,64,16384,0.057036801179250085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,64,6144,0.016793600718180337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,64,5120,0.015223466356595359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,64,3584,0.013175466656684875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,64,4096,0.013858133554458618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,64,3072,0.01225386659304301
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,64,2560,0.013141333063443502
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,64,1024,0.007884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,64,1536,0.009693866968154908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,64,768,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,64,2048,0.011366400122642516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,64,512,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,64,256,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,64,128,0.004130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,64,64,0.0034815999368826545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,64,32,0.0030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,128,65536,0.20305919647216797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,32,8192,0.027340799570083618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,32,10240,0.034030934174855546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,32,7168,0.017851734161376955
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,32,12288,0.03870720068613688
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,32,6144,0.01525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,32,16384,0.04952746629714966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,32,5120,0.013755733768145243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,32,4096,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,32,3072,0.011264000336329143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,32,3584,0.012049067020416259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,32,2560,0.012868266304334006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,32,2048,0.011127466956774395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,32,1024,0.007816533247629803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,32,768,0.006963199873765309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,32,1536,0.009489066402117411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,32,256,0.005085866649945577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,32,512,0.006109866499900818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,32,128,0.00365226666132609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,32,64,0.0034133332471052804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,32,32,0.0031744000812371576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,64,65536,0.18445653915405275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2048,32,65536,0.17264639536539714
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,65536,7168,3.685785675048828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,65536,8192,4.188228352864583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,65536,10240,5.259878540039063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,65536,5120,2.643182881673177
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,65536,6144,3.078382873535156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,65536,3584,1.8402645111083984
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,65536,3072,1.5745707194010417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,65536,1024,0.5637802759806315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,65536,2560,1.3081258138020835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,65536,768,0.4455423990885417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,65536,512,0.3089066823323568
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,65536,128,0.18438827196757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,65536,256,0.18773333231608075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,65536,1536,0.7918250401814778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,65536,64,0.13595306078592936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,65536,2048,1.0498389561971029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,65536,4096,2.048887507120768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,65536,32,0.10369706948598226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,16384,7168,0.883848508199056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,16384,12288,1.5563434600830077
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,65536,12288,6.192264302571615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,16384,8192,1.009117889404297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,16384,6144,0.7613781611124675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,16384,10240,1.2795221964518229
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,16384,5120,0.640614382425944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,16384,3072,0.3941376050313314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,16384,2048,0.2716671943664551
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,16384,2560,0.33242454528808596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,16384,4096,0.5128874778747559
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,16384,3584,0.4520277341206868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,16384,16384,2.046873601277669
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,16384,1536,0.21056853930155434
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,16384,1024,0.1486848036448161
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,16384,256,0.05280426740646362
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,16384,128,0.05044906536738077
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,16384,64,0.039185067017873124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,16384,768,0.11857919692993164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,16384,32,0.029320534070332843
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,16384,512,0.08840533097585043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,65536,16384,8.351402791341146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,12288,8192,0.8058538436889648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,12288,7168,0.701576550801595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,12288,12288,1.2060330708821616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,12288,6144,0.6092458724975586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,12288,10240,1.0036223729451499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,12288,5120,0.5093717257181803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,12288,3584,0.36355412801106773
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,12288,4096,0.4100778579711914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,12288,2048,0.21681493123372397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,12288,2560,0.2653866608937582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,12288,16384,1.5961429595947265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,12288,3072,0.3151189486185709
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,12288,768,0.09495893319447836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,12288,1024,0.11864746411641439
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,12288,1536,0.16807252566019695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,12288,512,0.07075839837392171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,12288,128,0.03925333420435588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,12288,256,0.04648960034052531
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,12288,64,0.029798400402069092
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,12288,32,0.02368853290875753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,10240,8192,0.6067541122436524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,10240,6144,0.4574890772501628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,16384,65536,8.614571126302083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,10240,10240,0.7400447845458984
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,10240,7168,0.5286911964416504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,10240,12288,0.9172992070515951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,10240,16384,1.23231569925944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,10240,4096,0.3126954714457194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,10240,5120,0.3843754768371582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,10240,3072,0.23719253540039062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,10240,3584,0.2721791903177897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,10240,2560,0.20227413177490233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,10240,1024,0.09181866645812989
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,10240,1536,0.12782933712005615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,10240,2048,0.16435200373331707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,10240,512,0.05382826725641886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,10240,768,0.07782399654388428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,10240,128,0.029764266808827718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,10240,256,0.035396265983581546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,10240,32,0.01911466717720032
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,10240,64,0.022835199038187662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,8192,7168,0.4729173342386882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,8192,12288,0.7822677612304687
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,8192,8192,0.5322410583496093
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,8192,10240,0.662391471862793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,8192,5120,0.3816447893778483
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,8192,6144,0.4549973487854004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,8192,16384,1.0318506876627604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,8192,4096,0.30750719706217444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,8192,3584,0.27248640060424806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,8192,3072,0.23582720756530762
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,8192,2560,0.19927040735880536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,8192,2048,0.16233812967936198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,12288,65536,6.239129638671875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,8192,768,0.07147520383199056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,8192,1536,0.12660053571065266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,8192,1024,0.08929279645284018
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,8192,512,0.05290666818618774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,8192,64,0.02222080032030741
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,8192,256,0.034815998872121175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,8192,128,0.028842665751775104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,8192,32,0.018500266472498576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,10240,65536,5.249809265136719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,7168,12288,0.7443114598592122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,7168,6144,0.40731306076049806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,7168,7168,0.46619307200113935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,7168,8192,0.5249365488688151
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,7168,10240,0.6476800282796223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,7168,16384,0.9805824279785156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,7168,5120,0.3795285224914551
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,7168,2048,0.16141653060913086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,7168,4096,0.3057322820027669
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,7168,3072,0.23296000162760416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,7168,3584,0.2698581377665202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,7168,2560,0.19763199488321942
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,7168,1536,0.1264298677444458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,7168,1024,0.08878080050150552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,7168,768,0.07055359681447347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,7168,512,0.05222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,7168,64,0.021811199188232423
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,7168,256,0.034167468547821045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,7168,128,0.027818665901819868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,7168,32,0.01802240014076233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,6144,7168,0.3525290807088216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,8192,65536,4.178500366210938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,6144,12288,0.5952170689900715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,6144,8192,0.4003498713175456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,6144,10240,0.49715201059977215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,6144,4096,0.20681385993957518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,6144,6144,0.3063466707865397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,6144,5120,0.2535765329996745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,6144,16384,0.7844181060791016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,6144,3584,0.1817599932352702
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,6144,3072,0.16080212593078613
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,6144,2560,0.13414400418599445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,6144,2048,0.10977280139923096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,6144,256,0.023995733261108397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,6144,512,0.03601066668828328
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,6144,768,0.048264535268147786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,6144,1024,0.06004053354263306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,6144,1536,0.08577706813812255
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,6144,128,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,6144,64,0.015735466281572977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,6144,32,0.012970667084058127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,5120,8192,0.39990612665812175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,5120,10240,0.397926394144694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,7168,65536,3.812317911783854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,5120,7168,0.35092480977376306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,5120,12288,0.4645546595255534
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,5120,16384,0.6127616246541341
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,5120,6144,0.30276266733805335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,5120,5120,0.2543274720509847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,5120,3072,0.15697919527689616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,5120,4096,0.20575572649637858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,5120,3584,0.1816234588623047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,5120,2560,0.13301760355631512
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,5120,2048,0.10929493109385173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,5120,1536,0.08512852986653646
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,5120,1024,0.05980159838994344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,5120,512,0.03577173153559367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,5120,768,0.0478549321492513
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,5120,256,0.023654399315516154
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,5120,128,0.01925119956334432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,5120,32,0.012868266304334006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,6144,65536,3.2024576822916666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,5120,64,0.015291733543078103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,4096,10240,0.36177921295166016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,4096,7168,0.28450133005777994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,4096,6144,0.30173867543538413
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,4096,8192,0.3190101305643717
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,4096,12288,0.4135594685872396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,4096,16384,0.5310805320739747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,4096,5120,0.2533034642537435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,4096,4096,0.20449280738830566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,4096,3072,0.15667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,4096,3584,0.18053119977315266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,4096,2560,0.132369065284729
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,4096,1024,0.05946026643117269
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,4096,1536,0.08410453001658122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,4096,2048,0.10796373685201008
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,4096,768,0.04778666496276855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,4096,64,0.015052800377209982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,4096,128,0.018943999210993448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,4096,256,0.022937599817911783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,4096,512,0.03546453317006429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,4096,32,0.012526933352152506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,5120,65536,2.6054656982421873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3584,8192,0.3174400011698405
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3584,12288,0.40857601165771484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3584,7168,0.28037118911743164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3584,10240,0.38731091817220054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3584,16384,0.5179050763448079
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3584,6144,0.24401920636494956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3584,4096,0.2036736011505127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3584,2560,0.13247146606445312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3584,3072,0.15592106183369953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3584,3584,0.17974613507588705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3584,5120,0.2526890595753988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3584,1536,0.08325119813283285
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3584,2048,0.10806612968444824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3584,1024,0.05946026643117269
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3584,512,0.03508906761805217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3584,768,0.04720640182495117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3584,128,0.018807466824849448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3584,64,0.015086932977040609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3584,32,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,4096,65536,2.1361663818359373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3584,256,0.022869332631429037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3072,8192,0.20104533831278482
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3072,6144,0.1516885280609131
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3072,12288,0.2960042635599772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3072,7168,0.17612800598144532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3072,10240,0.25118719736735023
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3072,5120,0.12840960025787354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3072,16384,0.39113388061523435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3072,3584,0.09195520083109537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3072,4096,0.1035264015197754
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3072,3072,0.07990612983703613
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3072,2560,0.06799360116322836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3072,2048,0.056012801329294835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3072,1536,0.04382719993591309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3072,1024,0.03174399932225545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3072,768,0.025873066981633504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3072,512,0.018739199638366698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3072,256,0.01262933313846588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3072,128,0.0106495996316274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3072,64,0.008703999718030294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3584,65536,1.882043711344401
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3072,32,0.007577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2560,10240,0.21585920651753746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2560,6144,0.1516544024149577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2560,7168,0.1749333381652832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2560,8192,0.1990997314453125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2560,12288,0.2507434686024984
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2560,5120,0.12810239791870118
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2560,16384,0.3249151865641276
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2560,4096,0.1033898671468099
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2560,2048,0.0556714653968811
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2560,3584,0.09137492974599203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2560,3072,0.07936000029246013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2560,2560,0.06778879960378012
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2560,1024,0.031402667363484696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2560,1536,0.04355413516362508
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2560,768,0.02474666635195414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2560,512,0.018397865692774455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2560,128,0.010478933652242024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2560,256,0.012356266379356384
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2560,32,0.00747519979874293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2560,64,0.008669867118199667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,3072,65536,1.561190414428711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2048,7168,0.13871787389119467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2048,6144,0.12209493319193523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2048,8192,0.15462400118509928
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2048,10240,0.1870848019917806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2048,12288,0.2174293359120687
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2048,16384,0.2822485287984212
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2048,5120,0.1060863971710205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2048,3584,0.0807253360748291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2048,3072,0.07297706604003906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2048,2560,0.0671061356862386
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2048,4096,0.09089706738789877
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2048,2048,0.05485226710637411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2560,65536,1.2821844736735026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2048,1024,0.030481066306432086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2048,512,0.018193066120147705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2048,256,0.012185600399971009
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2048,768,0.024439465999603272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2048,1536,0.042973868052164715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2048,128,0.010342400272687275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2048,64,0.008362666765848795
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2048,32,0.00730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1536,10240,0.1372159957885742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1536,8192,0.11298133532206218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1536,6144,0.08953173160552978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1536,7168,0.10158080259958904
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1536,12288,0.1613141377766927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1536,16384,0.20875946680704752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1536,4096,0.05485226710637411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1536,3584,0.04908373355865479
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1536,2560,0.03682986497879028
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1536,3072,0.042871467272440594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1536,5120,0.07703893184661866
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1536,2048,0.03078826665878296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,2048,65536,1.076258087158203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1536,1536,0.02481493353843689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1536,768,0.015428266922632852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1536,1024,0.01853440006573995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1536,512,0.01225386659304301
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1536,256,0.009523199995358785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1536,128,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1536,64,0.006075733403364817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1536,32,0.0053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1024,8192,0.08724479675292969
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1024,7168,0.08011093139648437
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1024,10240,0.10659840106964111
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1024,12288,0.11943253676096599
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1024,6144,0.07263573010762533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1024,16384,0.15261012713114422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1024,4096,0.054579198360443115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1024,3584,0.048401065667470294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1024,3072,0.0423253337542216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1024,2560,0.03642026583353679
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1024,5120,0.06649173498153686
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1024,2048,0.030173865954081218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1024,1024,0.01791999936103821
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1024,1536,0.024166399240493776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1024,512,0.01225386659304301
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1024,256,0.008840533097585042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1536,65536,0.7651669184366863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1024,768,0.01495039959748586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1024,128,0.007099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1024,32,0.0053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1024,64,0.005700266857941946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,768,8192,0.0630783994992574
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,768,7168,0.057207465171813965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,768,10240,0.07502506573994955
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,768,6144,0.050824534893035886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,768,12288,0.08758613268534342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,768,16384,0.12339200178782146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,768,4096,0.03904853264490764
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,768,3584,0.03290453354517619
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,768,5120,0.045021867752075194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,768,2560,0.02553173303604126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,768,3072,0.028706133365631104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,768,2048,0.020889600118001304
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,1024,65536,0.5498538970947265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,768,1536,0.016964266697565712
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,768,768,0.010922666390736897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,768,512,0.008942932883898417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,768,1024,0.012902399897575379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,768,256,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,768,128,0.0056320001681645715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,768,64,0.006041599810123444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,768,32,0.004710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,512,6144,0.03696639935175578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,512,8192,0.05287253459294637
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,512,7168,0.048401065667470294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,512,12288,0.06079146862030029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,512,10240,0.053452801704406736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,512,16384,0.07782399654388428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,512,5120,0.03256319959958394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,512,3584,0.02723840077718099
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,512,4096,0.028296534220377607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,512,3072,0.024098134040832518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,512,2048,0.017442133029301962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,512,2560,0.020753065745035805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,768,65536,0.4269397417704265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,512,1536,0.01437013347943624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,512,1024,0.010990933577219645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,512,768,0.00942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,512,256,0.006075733403364817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,512,512,0.007748266557852428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,512,64,0.004881066580613455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,512,32,0.003857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,512,128,0.005154133339722951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,256,7168,0.028262400627136232
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,256,10240,0.03614720106124878
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,256,8192,0.030856533845265703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,256,12288,0.03857066631317139
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,256,16384,0.049322664737701416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,256,6144,0.025668267409006757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,256,5120,0.022732800245285033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,256,3584,0.01935360034306844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,256,3072,0.015871999661127727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,256,4096,0.019831466674804687
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,512,65536,0.3058346748352051
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,256,2560,0.014506666858990987
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,256,2048,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,256,1536,0.010547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,256,768,0.0075434664885203045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,256,1024,0.008669867118199667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,256,256,0.0053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,256,512,0.006417066852251689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,256,128,0.004744533201058706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,256,32,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,256,64,0.0038912000755469004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,128,8192,0.02054826617240906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,128,12288,0.026316799720128375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,128,10240,0.023244800170262654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,128,7168,0.019080533583958944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,128,16384,0.03618133465449015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,128,6144,0.017373865842819212
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,128,5120,0.015837867061297098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,128,3072,0.012083199620246888
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,128,4096,0.014711466431617738
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,256,65536,0.15179093678792316
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,128,3584,0.012868266304334006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,128,2560,0.011229866743087768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,128,1024,0.007918933530648549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,128,2048,0.011366400122642516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,128,1536,0.009659733374913533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,128,768,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,128,128,0.0040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,128,512,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,128,256,0.005188266436258951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,128,64,0.0034133332471052804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,128,32,0.0030720000465710956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,64,10240,0.01686186591784159
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,64,8192,0.014847999811172486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,64,12288,0.017339734236399333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,64,7168,0.013516799608866373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,64,16384,0.029286400477091475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,64,6144,0.01242453356583913
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,128,65536,0.1113429307937622
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,64,3072,0.01160533328851064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,64,5120,0.011434666315714518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,64,3584,0.012390399972597759
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,64,4096,0.013209600249926248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,64,2560,0.010786133011182149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,64,1024,0.007816533247629803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,64,512,0.006075733403364817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,64,2048,0.008328533172607422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,64,768,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,64,1536,0.007850666840871174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,64,256,0.0044714664419492085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,64,128,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,64,64,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,64,32,0.0030037333567937215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,32,8192,0.013141333063443502
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,32,12288,0.01641813317934672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,32,10240,0.01454080045223236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,32,6144,0.011502933502197266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,32,7168,0.01225386659304301
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,32,16384,0.025600000222524004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,32,2560,0.008533333738644917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,32,5120,0.010547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,64,65536,0.10062506993611653
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,32,4096,0.009591466188430786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,32,3072,0.008942932883898417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,32,3584,0.009386666615804036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,32,2048,0.008055466910203297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,32,1536,0.0073728000124295544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,32,1024,0.0076117331782976794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,32,512,0.005905066430568695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,32,256,0.004539733131726583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,32,768,0.006758399804433187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,32,128,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,32,64,0.0032768001159032187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,32,32,0.004198400179545084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1024,32,65536,0.09171626567840577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,65536,5120,2.0765354156494142
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,65536,4096,1.637444305419922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,65536,7168,2.94468256632487
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,65536,3584,1.4298112233479818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,65536,3072,1.2125867207845054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,65536,2560,1.0234197616577148
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,65536,2048,0.8110421498616537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,65536,1536,0.6270975748697917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,65536,1024,0.44189014434814455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,65536,6144,2.4568491617838544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,65536,768,0.3586730639139811
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,65536,256,0.14632959365844728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,65536,64,0.105403733253479
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,65536,512,0.2443946679433187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,65536,32,0.08017919858296713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,65536,128,0.14755840301513673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,65536,8192,3.322675323486328
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,65536,10240,4.0919039408365885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,16384,12288,1.2312917073567708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,16384,16384,1.650551478068034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,16384,10240,1.0030421574910482
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,16384,8192,0.8159231821695963
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,16384,7168,0.6938282648722331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,16384,6144,0.6130346934000651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,65536,12288,4.96199696858724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,16384,5120,0.512716801961263
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,16384,1536,0.16909653345743816
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,16384,2560,0.26781012217203776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,16384,1024,0.12137813568115234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,16384,2048,0.21678080558776855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,16384,3072,0.3117055892944336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,16384,3584,0.3664213180541992
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,16384,768,0.09700693289438883
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,16384,4096,0.4107946713765462
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,16384,512,0.07157759666442871
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,16384,128,0.039185067017873124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,16384,256,0.04696746667226155
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,16384,64,0.03031040032704671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,16384,32,0.023517866929372154
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,65536,16384,6.662109883626302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,12288,7168,0.5316266695658366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,12288,8192,0.6052522659301758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,12288,6144,0.4568746566772461
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,12288,10240,0.7477930704752604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,12288,5120,0.384716796875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,12288,16384,1.213815434773763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,12288,12288,0.883848508199056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,12288,2048,0.16356693903605143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,12288,2560,0.20090880393981933
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,12288,1536,0.12690773010253906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,12288,3072,0.2354858716328939
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,12288,4096,0.3107839902242025
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,12288,3584,0.27333974838256836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,12288,1024,0.09117013613382975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,12288,768,0.07365972995758056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,12288,128,0.02935466567675273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,12288,256,0.034918399651845296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,12288,64,0.022528000672658286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,12288,512,0.05410133202870687
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,12288,32,0.01812480092048645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,16384,65536,6.536089579264323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,10240,7168,0.4727466583251953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,10240,6144,0.4127061208089192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,10240,8192,0.5250389417012532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,10240,5120,0.38034772872924805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,10240,10240,0.6326954523722331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,10240,4096,0.3088383992513021
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,10240,12288,0.7410687764485677
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,10240,3072,0.23490559260050453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,10240,3584,0.26951678593953454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,10240,2048,0.16186025937398274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,10240,2560,0.19927040735880536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,10240,16384,1.0000384012858072
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,10240,1536,0.1259178638458252
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,10240,512,0.05287253459294637
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,10240,1024,0.09099946816762289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,10240,768,0.07226026852925618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,10240,128,0.028910932938257854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,10240,256,0.03433813254038493
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,10240,32,0.018193066120147705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,10240,64,0.02222080032030741
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,8192,8192,0.40297813415527345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,8192,7168,0.350822385152181
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,8192,10240,0.4991317431131999
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,8192,6144,0.30453761418660485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,8192,12288,0.5977770487467449
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,8192,16384,0.7866367975870768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,8192,3584,0.18360320727030438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,8192,4096,0.20780372619628906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,8192,2560,0.13380266825358073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,8192,3072,0.15899306933085125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,12288,65536,4.755251057942709
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,8192,5120,0.25494186083475745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,8192,2048,0.11011412938435872
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,10240,65536,3.98919677734375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,8192,256,0.024439465999603272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,8192,1024,0.06154239972432455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,8192,512,0.036659200986226395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,8192,768,0.04915200074513753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,8192,1536,0.08594773610432943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,8192,128,0.019899733861287437
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,8192,32,0.013277866442998252
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,8192,64,0.015803733468055726
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,7168,8192,0.4017152150472005
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,7168,6144,0.304093869527181
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,7168,10240,0.4968106587727864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,7168,7168,0.3506858507792155
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,7168,12288,0.5961386362711589
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,7168,5120,0.25535146395365393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,7168,4096,0.20654080708821615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,7168,16384,0.7848960240681966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,7168,2048,0.1096021334330241
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,7168,3584,0.18216959635416669
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,7168,3072,0.15769599278767904
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,7168,2560,0.13359786669413248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,7168,1536,0.08536746501922607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,7168,768,0.048878931999206544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,7168,1024,0.06157653331756592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,8192,65536,3.207406870524088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,7168,512,0.03580586512883504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,7168,256,0.023654399315516154
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,7168,64,0.01525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,7168,128,0.01914880077044169
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,7168,32,0.01269760032494863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,6144,6144,0.3030698776245117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,6144,8192,0.30344533920288086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,6144,7168,0.3512319882710775
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,6144,10240,0.36433919270833337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,6144,16384,0.5689685185750325
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,6144,5120,0.2542933305104574
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,6144,12288,0.4344490687052409
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,6144,4096,0.2055509408315023
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,6144,2560,0.1334272066752116
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,6144,2048,0.10898773670196533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,6144,1024,0.060893865426381436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,6144,3584,0.18135040601094563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,6144,1536,0.08506027062733969
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,6144,3072,0.1573205312093099
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,6144,768,0.04768426815668742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,6144,512,0.03553280035654704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,6144,256,0.02307413419087728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,6144,128,0.01904639999071757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,6144,64,0.015223466356595359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,6144,32,0.012663466731707254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,5120,12288,0.4147199948628743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,5120,16384,0.5249365488688151
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,7168,65536,2.8037460327148436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,5120,8192,0.3195904095967611
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,5120,7168,0.282965342203776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,5120,10240,0.3935231844584147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,5120,5120,0.25371306737264
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,5120,6144,0.24777386983235677
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,5120,4096,0.20415147145589194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,5120,3072,0.15667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,5120,2560,0.1322325309117635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,5120,2048,0.10861226717631023
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,5120,3584,0.18046293258666993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,5120,1536,0.08454826672871908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,5120,256,0.023040000597635904
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,5120,1024,0.05946026643117269
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,5120,512,0.035259731610616046
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,5120,768,0.04751360019048055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,5120,128,0.019012266397476198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,5120,64,0.015086932977040609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,5120,32,0.01256106694539388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,6144,65536,2.344038391113281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,4096,8192,0.19988479614257812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,4096,7168,0.17640105883280438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,4096,10240,0.2481152057647705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,4096,12288,0.2965162595113119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,4096,6144,0.1516885280609131
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,4096,4096,0.1040725310643514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,4096,5120,0.12772693634033203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,4096,16384,0.3927040100097656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,4096,3584,0.09198933442433675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,4096,3072,0.07966720263163249
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,4096,2560,0.06778879960378012
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,5120,65536,1.9688448588053387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,4096,2048,0.05604693492253622
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,4096,1536,0.04379306634267171
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,4096,1024,0.031778132915496825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,4096,768,0.025873066981633504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,4096,512,0.01890986760457357
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,4096,64,0.008738133311271667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,4096,256,0.012526933352152506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,4096,128,0.010615467031796774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,4096,32,0.007441066702206929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3584,6144,0.1514154593149821
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3584,7168,0.17568426132202147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3584,10240,0.24821759859720866
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3584,8192,0.19933867454528809
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3584,12288,0.29617493947347007
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3584,5120,0.12741973400115966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3584,3584,0.09202346801757813
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3584,16384,0.392192014058431
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3584,4096,0.10376533667246501
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3584,3072,0.07946240107218425
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3584,1536,0.043724799156188966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3584,2048,0.05570559899012247
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3584,2560,0.06782293319702148
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3584,1024,0.0315391997496287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3584,768,0.02488320072491964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3584,256,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3584,128,0.010513066252072652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3584,512,0.01867093245188395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3584,64,0.008601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,4096,65536,1.5149055480957032
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3584,32,0.007441066702206929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3072,12288,0.2214911937713623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3072,16384,0.28876800537109376
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3072,7168,0.1404586633046468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3072,6144,0.1224021355311076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3072,8192,0.15499946276346843
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3072,10240,0.18851839701334636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3072,4096,0.09253546396891275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3072,5120,0.10762240091959636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3072,3584,0.08277333577473958
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3072,3072,0.079257599512736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3072,2560,0.06734506289164224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3072,2048,0.05512533187866211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3072,1536,0.043383467197418216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3584,65536,1.344034194946289
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3072,1024,0.03065173427263896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3072,768,0.024576000372568765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3072,512,0.018329600493113198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3072,256,0.01225386659304301
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3072,128,0.010478933652242024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3072,64,0.008567466338475545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3072,32,0.0073728000124295544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2560,7168,0.14008320172627767
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2560,8192,0.15383893648783367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2560,6144,0.12124160130818684
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2560,10240,0.18766506512959796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2560,12288,0.21783893903096518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2560,16384,0.2824533462524414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2560,5120,0.10871466795603435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2560,3072,0.07270399729410806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2560,3584,0.08116906483968099
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2560,4096,0.08994133472442627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2560,2560,0.06703786849975586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,3072,65536,1.1108010609944663
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2560,2048,0.05485226710637411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2560,1536,0.043144532044728595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2560,256,0.012117333213488261
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2560,1024,0.03051519989967346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2560,512,0.018158932526906334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2560,768,0.024541866779327393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2560,32,0.00730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2560,64,0.008499200145403545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2560,128,0.010410666465759277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2048,6144,0.0892245372136434
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2048,8192,0.11359573205312093
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2048,7168,0.10178559621175129
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2048,10240,0.14110719362894694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2048,12288,0.16117760340372722
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2048,16384,0.20920319557189943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2048,4096,0.05519359906514486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2048,3584,0.04911786715189616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2048,5120,0.0772437334060669
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2048,3072,0.042973868052164715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2048,2560,0.03686399857203166
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2048,2048,0.030754133065541582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2560,65536,0.939246940612793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2048,1536,0.02491733431816101
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2048,1024,0.018602667252222697
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2048,512,0.012288000186284382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2048,256,0.00962559978167216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2048,768,0.015530666708946228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2048,128,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2048,32,0.0052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2048,64,0.006007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1536,7168,0.08069120248158773
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1536,6144,0.07219200134277344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1536,10240,0.10581333637237549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1536,8192,0.08796160221099854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1536,12288,0.12264106273651124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1536,16384,0.15438507397969564
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1536,5120,0.06693546772003174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1536,4096,0.054579198360443115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1536,3584,0.048640000820159915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1536,3072,0.04256426493326823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1536,2560,0.03645439942677815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1536,1536,0.024439465999603272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1536,2048,0.03041279911994934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,2048,65536,0.7463594436645508
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1536,1024,0.01812480092048645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1536,512,0.011912533640861511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1536,768,0.015121066570281982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1536,256,0.008942932883898417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1536,128,0.007133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1536,64,0.005700266857941946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1536,32,0.005085866649945577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1024,8192,0.06376106739044189
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1024,12288,0.08700586954752604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1024,7168,0.05683199961980184
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1024,6144,0.05116586685180664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1024,10240,0.07519573370615641
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1024,16384,0.1301162640253703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1024,5120,0.04522666533788045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1536,65536,0.5463722864786784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1024,3072,0.028467200199762982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1024,4096,0.03863893349965413
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1024,2560,0.024985599517822265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1024,3584,0.03304106593132019
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1024,2048,0.020855466524759926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1024,1536,0.017100799083709716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1024,1024,0.013004799683888754
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1024,768,0.01088853379090627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1024,256,0.006690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1024,64,0.006007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1024,128,0.0056320001681645715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1024,512,0.00890880028406779
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1024,32,0.004744533201058706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,768,6144,0.04498773415883382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,768,8192,0.05355519851048788
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,768,7168,0.04942506551742554
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,768,10240,0.05369173288345337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,768,12288,0.06133759816487631
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,768,16384,0.08161280155181885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,1024,65536,0.43264001210530595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,768,5120,0.03310933311780294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,768,3072,0.024337067206700643
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,768,4096,0.028637866179148357
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,768,3584,0.02768213351567586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,768,2560,0.021026132504145305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,768,2048,0.017749333381652833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,768,1536,0.01454080045223236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,768,1024,0.01129813293615977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,768,768,0.009659733374913533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,768,512,0.00795306662718455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,768,128,0.0053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,768,256,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,768,64,0.005154133339722951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,768,32,0.004198400179545084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,512,8192,0.03583999872207642
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,512,7168,0.032631466786066696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,512,10240,0.04194986820220947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,512,12288,0.048128000895182294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,512,6144,0.02921813329060872
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,512,16384,0.06007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,768,65536,0.38075733184814453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,512,5120,0.026146133740743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,512,4096,0.023108265797297158
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,512,3584,0.021504000822703043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,512,3072,0.020002132654190062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,512,2048,0.015155200163523355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,512,2560,0.018261333306630455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,512,1536,0.012834133704503379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,512,768,0.008738133311271667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,512,512,0.007577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,512,256,0.00580266664425532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,512,128,0.005085866649945577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,512,64,0.004198400179545084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,512,1024,0.010103467106819152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,512,32,0.0034815999368826545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,256,8192,0.02239146629969279
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,256,10240,0.02590720057487488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,256,12288,0.029627732435862225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,512,65536,0.20920319557189943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,256,6144,0.01904639999071757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,256,16384,0.0373418649037679
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,256,7168,0.020718934138615926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,256,4096,0.015803733468055726
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,256,3072,0.014847999811172486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,256,3584,0.017032533884048462
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,256,5120,0.017544533809026083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,256,2560,0.0130730668703715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,256,2048,0.011264000336329143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,256,1536,0.00955733358860016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,256,1024,0.007918933530648549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,256,768,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,256,128,0.0044714664419492085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,256,512,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,256,256,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,256,64,0.003583999971548716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,256,32,0.00314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,128,7168,0.014574933052062988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,128,8192,0.01641813317934672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,128,10240,0.01771519978841146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,128,12288,0.019217065970102944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,128,16384,0.029252266883850096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,256,65536,0.11352746486663819
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,128,6144,0.013414399822553
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,128,5120,0.01259519954522451
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,128,3584,0.011707733074824016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,128,3072,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,128,2560,0.010513066252072652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,128,4096,0.011195733149846395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,128,2048,0.009215999643007915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,128,256,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,128,512,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,128,768,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,128,1024,0.007884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,128,1536,0.00962559978167216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,128,128,0.003618133316437403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,128,64,0.0031744000812371576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,128,32,0.00314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,64,7168,0.011229866743087768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,64,8192,0.012663466731707254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,64,12288,0.01460906664530436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,64,6144,0.010513066252072652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,128,65536,0.08584533532460531
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,64,16384,0.017339734236399333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,64,10240,0.013209600249926248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,64,5120,0.009966933727264404
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,64,4096,0.009113599856694538
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,64,3584,0.008942932883898417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,64,3072,0.008601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,64,2560,0.008294400076071422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,64,2048,0.007816533247629803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,64,1536,0.007441066702206929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,64,256,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,64,1024,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,64,512,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,64,128,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,64,64,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,64,768,0.006894933183987935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,64,32,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,32,7168,0.010001066327095031
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,32,8192,0.011127466956774395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,32,10240,0.01153706709543864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,32,12288,0.01259519954522451
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,64,65536,0.07335253556569418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,32,16384,0.015223466356595359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,32,6144,0.009386666615804036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,32,4096,0.008396800359090168
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,32,3584,0.008191999793052674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,32,5120,0.009011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,32,3072,0.007884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,32,2048,0.007167999943097432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,32,2560,0.007714133461316426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,32,1536,0.006929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,32,768,0.006758399804433187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,32,1024,0.006621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,32,256,0.004266666869322458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,32,512,0.005870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,32,128,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,32,64,0.004095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,32,32,0.004232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,768,32,65536,0.07379626433054606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,65536,4096,1.0532864252726237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,65536,5120,1.32498779296875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,65536,8192,2.1763755798339846
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,65536,3072,0.7927125295003254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,65536,3584,0.925320561726888
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,65536,2560,0.6695253372192382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,65536,1536,0.41936213175455733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,65536,6144,1.5533397674560547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,65536,2048,0.5416277567545573
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,65536,1024,0.297267214457194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,65536,7168,1.894980239868164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,65536,768,0.23541760444641113
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,65536,256,0.09925973415374756
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,65536,32,0.05676373243331909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,65536,128,0.0991914669672648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,65536,512,0.15981225967407225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,65536,64,0.0748202641805013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,16384,12288,0.78919677734375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,16384,16384,1.044753074645996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,16384,8192,0.5347328186035156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,16384,10240,0.645631980895996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,65536,10240,2.6678614298502605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,16384,7168,0.4750677426656087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,16384,6144,0.4565333366394043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,16384,3072,0.23678293228149414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,16384,2560,0.20032854080200196
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,16384,4096,0.3051520029703776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,16384,5120,0.3822250684102376
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,65536,12288,3.288371276855469
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,16384,3584,0.2714282671610514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,16384,2048,0.16278187433878583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,16384,1536,0.1268053372701009
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,16384,1024,0.09007786909739177
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,16384,768,0.07205546696980794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,16384,256,0.034815998872121175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,16384,512,0.05355519851048788
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,16384,128,0.0291157325108846
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,16384,64,0.02228906750679016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,16384,32,0.01812480092048645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,12288,8192,0.4031488100687663
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,12288,10240,0.5002581278483074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,65536,16384,4.338858540852865
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,12288,7168,0.3482624053955078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,12288,6144,0.3051520029703776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,12288,5120,0.25644373893737793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,12288,12288,0.5972991943359375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,12288,3072,0.15875412623087565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,12288,4096,0.20626773834228515
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,12288,16384,0.789469846089681
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,12288,2560,0.1343488057454427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,12288,3584,0.18121387163798014
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,12288,2048,0.10980693499247234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,12288,1536,0.08622079690297445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,12288,512,0.03621546824773152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,12288,768,0.04894719918568929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,12288,1024,0.06113280057907104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,12288,256,0.027511467536290485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,12288,64,0.015735466281572977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,12288,128,0.020002132654190062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,12288,32,0.012936533490816752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,16384,65536,4.350395711263021
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,10240,7168,0.3526314735412598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,10240,10240,0.3995306650797526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,10240,4096,0.2063701311747233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,10240,5120,0.255078395207723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,10240,6144,0.3033770561218262
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,10240,8192,0.4008960088094075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,10240,12288,0.46315520604451493
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,10240,3584,0.1857877254486084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,10240,3072,0.15779840151468913
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,10240,16384,0.6139904022216797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,10240,2560,0.1337685267130534
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,10240,1536,0.08526506423950195
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,10240,1024,0.06621866623560588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,10240,2048,0.10919253031412761
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,10240,768,0.04833279848098755
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,10240,64,0.015291733543078103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,10240,512,0.03594239950180054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,10240,256,0.02379093368848165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,10240,128,0.019694934288660683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,10240,32,0.012731732924779257
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,8192,10240,0.36218878428141277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,8192,16384,0.5392725626627605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,8192,8192,0.3168938636779785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,8192,7168,0.2846037228902181
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,8192,12288,0.41216001510620115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,8192,5120,0.2546346664428711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,8192,6144,0.3019434611002604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,12288,65536,3.2878590901692712
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,8192,2560,0.13305173714955648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,8192,3072,0.15680853525797528
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,8192,4096,0.2044245402018229
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,8192,2048,0.10820266405741374
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,8192,3584,0.1810431957244873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,8192,1536,0.0859818696975708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,8192,1024,0.06007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,8192,768,0.04792319933573405
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,8192,512,0.03570346832275391
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,8192,256,0.02362026572227478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,8192,128,0.01890986760457357
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,8192,32,0.01259519954522451
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,8192,64,0.015121066570281982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,7168,8192,0.3182933489481608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,7168,12288,0.40861012140909836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,7168,7168,0.2800981203715006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,7168,10240,0.39051946004231775
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,10240,65536,2.6484395345052083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,7168,6144,0.2440874735514323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,8192,65536,2.200098164876302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,7168,16384,0.5245269457499186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,7168,5120,0.2525866667429606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,7168,2560,0.13226666450500488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,7168,2048,0.10949973265329997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,7168,4096,0.2041173299153646
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,7168,3072,0.15588693618774413
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,7168,3584,0.18008747100830078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,7168,1024,0.059665067990620935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,7168,768,0.04778666496276855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,7168,1536,0.0861525297164917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,7168,512,0.03549866676330567
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,7168,256,0.023347200949986775
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,7168,128,0.01884160041809082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,7168,64,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,7168,32,0.01242453356583913
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,6144,6144,0.15189332962036134
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,6144,12288,0.2968576113382975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,6144,7168,0.1764352003733317
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,6144,8192,0.19967999458312988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,6144,10240,0.2481152057647705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,6144,4096,0.10379947026570638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,6144,5120,0.12803413073221842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,6144,16384,0.3906559944152832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,6144,2048,0.05587626695632934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,6144,2560,0.06789120038350424
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,6144,1536,0.044100264708201095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,6144,3072,0.07976960341135661
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,6144,3584,0.0920576016108195
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,6144,1024,0.031846400101979574
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,6144,768,0.025941334168116253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,6144,512,0.019080533583958944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,6144,256,0.01269760032494863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,6144,128,0.010683733224868774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,6144,64,0.008738133311271667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,6144,32,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,7168,65536,1.9442347208658854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,5120,12288,0.2525866667429606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,5120,7168,0.1749333381652832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,5120,8192,0.199509334564209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,5120,10240,0.2470911979675293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,5120,6144,0.15182506243387858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,6144,65536,1.5580501556396484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,5120,16384,0.32614399592081705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,5120,5120,0.12755626837412518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,5120,4096,0.10311679840087891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,5120,3072,0.07949653466542562
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,5120,2048,0.055569068590799964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,5120,3584,0.09147733052571615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,5120,2560,0.06775466601053873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,5120,1536,0.043622398376464845
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,5120,1024,0.030822400252024335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,5120,768,0.02488320072491964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,5120,256,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,5120,512,0.018739199638366698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,5120,64,0.008567466338475545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,5120,128,0.010513066252072652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,5120,32,0.0073728000124295544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,4096,12288,0.2194431940714518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,4096,7168,0.13977600733439127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,4096,6144,0.1218559980392456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,4096,8192,0.1547946612040202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,4096,10240,0.18763093948364257
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,4096,16384,0.28231681187947594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,4096,5120,0.10844159921010335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,4096,3584,0.08161280155181885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,4096,3072,0.07365972995758056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,4096,2560,0.06700373490651448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,4096,4096,0.09055573145548503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,5120,65536,1.2944042205810546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,4096,2048,0.05492053429285685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,4096,1536,0.043178665637969973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,4096,768,0.024576000372568765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,4096,1024,0.030583467086156207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,4096,512,0.01843199928601583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,4096,256,0.012185600399971009
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,4096,32,0.007236266632874806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,4096,64,0.008396800359090168
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,4096,128,0.010308266679445902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3584,8192,0.15428266525268555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3584,6144,0.12038826942443848
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3584,7168,0.13837653795878094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3584,10240,0.18691412607828778
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3584,12288,0.2168490727742513
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3584,16384,0.2838186581929525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3584,5120,0.10605226357777912
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,4096,65536,1.0844842274983724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3584,4096,0.0892245372136434
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3584,3584,0.08062293529510497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3584,2560,0.06700373490651448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3584,3072,0.07253333727518717
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3584,1536,0.04304213523864746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3584,1024,0.03054933349291484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3584,2048,0.05488640069961548
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3584,768,0.024541866779327393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3584,64,0.008430932958920796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3584,512,0.0180565337340037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3584,32,0.007099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3584,256,0.012083199620246888
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3584,128,0.01027413308620453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3072,8192,0.11322027047475178
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3072,12288,0.1617919921875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3072,6144,0.08901973565419516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3072,7168,0.10164906978607177
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3072,10240,0.1435647964477539
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3072,16384,0.21145599683125815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3072,5120,0.07758506933848063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3072,4096,0.05529599984486898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3072,2048,0.030822400252024335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3072,3584,0.04901546637217204
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3072,2560,0.03689813216527303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3072,3072,0.042973868052164715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3072,1536,0.02491733431816101
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3072,1024,0.018739199638366698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3584,65536,0.9273685455322266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3072,64,0.006109866499900818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3072,32,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3072,256,0.009454933802286784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3072,128,0.0075434664885203045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3072,512,0.012356266379356384
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3072,768,0.015701333681742348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2560,6144,0.07895039717356364
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2560,12288,0.16950613657633465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2560,7168,0.10011306603749592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2560,8192,0.11226452986399334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2560,10240,0.1376255989074707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2560,16384,0.18660693168640136
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,3072,65536,0.7558826446533203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2560,4096,0.0549888014793396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2560,5120,0.06683306694030762
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2560,2048,0.03061760067939758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2560,2560,0.036659200986226395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2560,3072,0.0425983985265096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2560,3584,0.04870826800664266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2560,1024,0.018500266472498576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2560,512,0.012049067020416259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2560,768,0.015359999736150107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2560,1536,0.024576000372568765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2560,256,0.008942932883898417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2560,128,0.00730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2560,64,0.005870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2560,32,0.005120000243186951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2048,6144,0.07099733352661133
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2048,7168,0.07936000029246013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2048,10240,0.10523306528727214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2048,12288,0.12008106708526611
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2048,8192,0.08898560206095377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2048,16384,0.1541802724202474
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2048,5120,0.06669653256734212
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2560,65536,0.6234794616699219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2048,3584,0.04833279848098755
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2048,3072,0.042393600940704344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2048,4096,0.05437440077463786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2048,2560,0.03631786505381267
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2048,1536,0.024268800020217897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2048,2048,0.03020799954732259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2048,1024,0.018158932526906334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2048,512,0.012151466806729634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2048,768,0.015052800377209982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2048,128,0.007099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2048,256,0.008840533097585042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2048,64,0.005700266857941946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2048,32,0.005358933409055074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1536,7168,0.057309865951538086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1536,8192,0.06283946832021078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1536,10240,0.07522986729939779
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1536,12288,0.08714240392049154
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1536,6144,0.050824534893035886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1536,16384,0.12731733322143554
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1536,5120,0.04526079893112182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1536,3072,0.028603732585906982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1536,4096,0.03857066631317139
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1536,3584,0.03607893387476603
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,2048,65536,0.5582848230997721
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1536,2560,0.02505386670430501
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1536,1536,0.017339734236399333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1536,2048,0.020855466524759926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1536,128,0.0055637334783871974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1536,1024,0.012970667084058127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1536,512,0.008840533097585042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1536,768,0.010990933577219645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1536,64,0.006075733403364817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1536,256,0.006690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1536,32,0.004744533201058706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1024,8192,0.05314559936523437
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1024,10240,0.052940801779429114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1536,65536,0.42588160832722977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1024,6144,0.03689813216527303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1024,12288,0.0606549342473348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1024,7168,0.04894719918568929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1024,16384,0.07830186684926352
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1024,5120,0.03239253362019857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1024,3072,0.024132267634073893
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1024,1536,0.01437013347943624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1024,3584,0.02744320034980774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1024,2560,0.020889600118001304
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1024,4096,0.028330665826797486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1024,2048,0.017646932601928712
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1024,1024,0.011025066177050274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1024,512,0.007782400151093801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1024,768,0.00942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1024,256,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1024,128,0.005188266436258951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1024,64,0.004846933484077454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1024,32,0.0038912000755469004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,768,8192,0.03573760191599528
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,768,7168,0.032529066006342575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,768,10240,0.04212053219477336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,768,12288,0.048264535268147786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,1024,65536,0.31105705897013347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,768,16384,0.05997226635615031
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,768,6144,0.029422932863235475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,768,3072,0.020070399840672812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,768,3584,0.021640533208847047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,768,4096,0.023005867004394533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,768,2560,0.018568533658981323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,768,5120,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,768,2048,0.015223466356595359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,768,1536,0.012663466731707254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,768,1024,0.009966933727264404
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,768,768,0.008703999718030294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,768,512,0.007645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,768,64,0.004334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,768,128,0.0053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,768,256,0.00576853354771932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,768,65536,0.20930560429890951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,768,32,0.003549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,512,7168,0.02788693308830261
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,512,8192,0.03031040032704671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,512,10240,0.035327998797098796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,512,12288,0.03679573138554891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,512,16384,0.046114134788513186
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,512,6144,0.025804799795150758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,512,3072,0.016008533040682473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,512,3584,0.019387733936309815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,512,2560,0.01443839967250824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,512,4096,0.020241065820058187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,512,5120,0.02263039946556091
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,512,2048,0.01242453356583913
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,512,1536,0.0105813334385554
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,512,1024,0.008635733524958293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,512,768,0.00747519979874293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,512,512,0.006485333542029063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,512,32,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,512,256,0.005256533126036326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,512,64,0.0038912000755469004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,512,128,0.004710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,256,7168,0.01914880077044169
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,512,65536,0.14605654080708821
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,256,8192,0.02065066695213318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,256,12288,0.022050132354100548
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,256,6144,0.017442133029301962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,256,16384,0.02959360082944234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,256,10240,0.023005867004394533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,256,4096,0.014745600024859109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,256,5120,0.015837867061297098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,256,3072,0.012083199620246888
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,256,2560,0.011264000336329143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,256,3584,0.012868266304334006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,256,2048,0.010342400272687275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,256,256,0.005188266436258951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,256,1024,0.007884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,256,768,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,256,512,0.006178133189678192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,256,128,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,256,64,0.0033791999022165934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,256,1536,0.009659733374913533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,256,32,0.0032426667710145317
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,256,65536,0.08628906408945719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,128,8192,0.013823999961217245
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,128,12288,0.017851734161376955
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,128,10240,0.015223466356595359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,128,16384,0.021128533283869426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,128,6144,0.011912533640861511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,128,5120,0.011195733149846395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,128,7168,0.012834133704503379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,128,4096,0.010137599706649781
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,128,2560,0.00890880028406779
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,128,3584,0.009796266754468281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,128,3072,0.009352533022562663
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,128,2048,0.008294400076071422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,128,1536,0.00795306662718455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,128,768,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,128,512,0.006041599810123444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,128,1024,0.007816533247629803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,128,256,0.0045056000351905824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,128,64,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,128,128,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,128,32,0.0030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,128,65536,0.06813013553619385
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,64,8192,0.011366400122642516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,64,10240,0.011946666240692138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,64,6144,0.009215999643007915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,64,7168,0.010137599706649781
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,64,16384,0.01443839967250824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,64,5120,0.008840533097585042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,64,12288,0.012663466731707254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,64,4096,0.008191999793052674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,64,3584,0.007987200220425924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,64,3072,0.007714133461316426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,64,2048,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,64,2560,0.007406933108965556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,64,1536,0.006792533397674561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,64,768,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,64,256,0.004334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,64,512,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,64,128,0.0032768001159032187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,64,32,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,64,1024,0.0065194666385650635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,64,64,0.0033791999022165934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,64,65536,0.06311253309249878
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,32,8192,0.010035199920336406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,32,12288,0.011229866743087768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,32,6144,0.008772266904513042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,32,10240,0.010342400272687275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,32,16384,0.01269760032494863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,32,7168,0.009386666615804036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,32,5120,0.008328533172607422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,32,2560,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,32,4096,0.007816533247629803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,32,3584,0.007645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,32,3072,0.00730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,32,2048,0.006826666494210561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,32,1024,0.00631466656923294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,32,1536,0.006621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,32,768,0.006690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,32,512,0.005870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,32,64,0.004095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,32,256,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,32,128,0.0034815999368826545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,32,32,0.004130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,512,32,65536,0.052121599515279136
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,65536,4096,0.829644775390625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,65536,6144,1.2563114166259766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,65536,3584,0.729361089070638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,65536,3072,0.6332757314046223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,65536,5120,1.021951993306478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,65536,2048,0.44533761342366535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,65536,2560,0.5304320017496745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,65536,768,0.19131733576456705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,65536,1024,0.24077653884887695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,65536,7168,1.4661632537841798
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,65536,8192,1.652872594197591
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,65536,256,0.07823359966278076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,65536,512,0.13987840016682943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,65536,16384,3.452177174886068
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,65536,32,0.0444757342338562
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,65536,1536,0.34696534474690754
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,65536,128,0.08038400014241537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,65536,64,0.062395731608072914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,65536,10240,2.12937380472819
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,16384,12288,0.5989376068115234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,16384,16384,0.7962624231974285
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,16384,7168,0.3485013326009115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,16384,6144,0.30566399892171225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,16384,8192,0.40328534444173175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,65536,12288,2.5421483357747396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,16384,10240,0.4971178690592448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,16384,5120,0.2570240020751953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,16384,4096,0.2081450621287028
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,16384,3584,0.18138453165690105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,16384,2048,0.11499520142873128
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,16384,3072,0.1588223934173584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,16384,2560,0.1344853401184082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,16384,1536,0.09110186894734701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,16384,1024,0.06382933457692465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,16384,512,0.037307735284169516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,16384,128,0.02065066695213318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,16384,64,0.01576959987481435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,16384,32,0.013004799683888754
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,16384,256,0.024541866779327393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,16384,768,0.050722134113311765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,12288,6144,0.3041621208190918
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,12288,5120,0.2551808039347331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,12288,7168,0.3501738548278809
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,12288,8192,0.3014997482299805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,12288,10240,0.3703808148701986
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,12288,12288,0.435916805267334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,12288,4096,0.20630186398824057
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,12288,16384,0.5558272043863932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,12288,3584,0.18145279884338378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,12288,3072,0.1571839968363444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,12288,2048,0.11192320187886555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,12288,2560,0.13370025952657064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,12288,1024,0.06143999894460043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,12288,1536,0.08656213283538819
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,12288,768,0.04911786715189616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,12288,512,0.03597653309504191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,12288,256,0.024098134040832518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,12288,128,0.019797333081563315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,12288,64,0.01532586713631948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,12288,32,0.013038933277130127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,10240,7168,0.28419411977132164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,16384,65536,3.3243476867675783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,10240,8192,0.317849604288737
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,10240,10240,0.39748268127441405
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,10240,12288,0.4135253270467122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,10240,5120,0.25415679613749187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,10240,6144,0.24832000732421874
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,10240,16384,0.5341525395711263
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,10240,2048,0.1099776029586792
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,10240,4096,0.20377599398295082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,10240,2560,0.132369065284729
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,10240,3072,0.15670612653096516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,10240,3584,0.1809066613515218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,12288,65536,2.42513910929362
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,10240,1536,0.08645973205566407
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,10240,1024,0.061166934172312414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,10240,512,0.03563520113627116
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,10240,768,0.048503466447194415
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,10240,256,0.0237226665019989
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,10240,128,0.01925119956334432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,10240,64,0.015291733543078103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,10240,32,0.01262933313846588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,8192,7168,0.17694719632466632
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,8192,16384,0.39365971883138023
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,8192,6144,0.1519957383473714
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,8192,8192,0.20196693738301597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,8192,10240,0.2482858657836914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,8192,12288,0.2952191988627116
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,8192,5120,0.1287168025970459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,8192,4096,0.10926079750061035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,8192,1536,0.04686506589253743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,8192,2048,0.058641068140665685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,8192,2560,0.06830080350240073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,8192,3584,0.09212586879730225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,8192,3072,0.08052053451538085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,8192,768,0.025565866629282636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,8192,1024,0.032290132840474446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,8192,512,0.01949013272921244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,8192,256,0.013482667009035745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,8192,64,0.009215999643007915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,8192,128,0.011229866743087768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,8192,32,0.007918933530648549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,7168,7168,0.17547945976257323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,10240,65536,1.9954346974690753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,7168,8192,0.1994069258371989
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,7168,10240,0.24750080108642578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,7168,12288,0.2958677291870117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,7168,6144,0.15155199368794758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,8192,65536,1.71680425008138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,7168,16384,0.39041706720987956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,7168,2560,0.0676522652308146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,7168,5120,0.12796586354573566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,7168,4096,0.10335573355356853
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,7168,3584,0.09161386489868165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,7168,3072,0.07970133622487387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,7168,1536,0.044544001420338944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,7168,2048,0.056695465246836343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,7168,1024,0.030958932638168336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,7168,512,0.018807466824849448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,7168,128,0.010513066252072652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,7168,256,0.01259519954522451
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,7168,768,0.024985599517822265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,7168,64,0.008567466338475545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,7168,32,0.007441066702206929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,6144,6144,0.12318720022837322
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,6144,7168,0.14093653361002606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,6144,8192,0.15561386744181316
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,6144,10240,0.1912490685780843
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,6144,12288,0.2214570681254069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,6144,5120,0.10837333202362061
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,7168,65536,1.412164306640625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,6144,16384,0.28706134160359703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,6144,4096,0.09253546396891275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,6144,3584,0.0839680035909017
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,6144,2560,0.067413330078125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,6144,3072,0.07929173310597738
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,6144,1536,0.043383467197418216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,6144,1024,0.030754133065541582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,6144,2048,0.05550080140431722
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,6144,512,0.01867093245188395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,6144,768,0.02474666635195414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,6144,256,0.012356266379356384
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,6144,128,0.010444800059000652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,6144,64,0.008533333738644917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,6144,32,0.0073728000124295544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,5120,7168,0.13861546516418458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,5120,8192,0.15616000493367513
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,5120,10240,0.1914197285970052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,5120,12288,0.21705387433369955
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,5120,6144,0.1217194636662801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,5120,16384,0.28388694127400715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,6144,65536,1.1224746704101562
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,5120,3072,0.07328426837921143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,5120,2560,0.0671061356862386
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,5120,5120,0.10543786684672038
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,5120,3584,0.08185173670450846
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,5120,4096,0.09106773535410563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,5120,2048,0.05488640069961548
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,5120,1536,0.04266666571299235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,5120,1024,0.030583467086156207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,5120,768,0.024576000372568765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,5120,256,0.012117333213488261
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,5120,128,0.010410666465759277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,5120,512,0.018500266472498576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,5120,64,0.00846506655216217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,5120,32,0.00730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,4096,6144,0.08970239957173666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,4096,8192,0.11315200328826905
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,4096,7168,0.10134186744689941
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,4096,10240,0.13738667170206706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,4096,12288,0.16127999623616535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,5120,65536,0.9609557469685873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,4096,5120,0.08260266780853272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,4096,16384,0.21978453000386558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,4096,4096,0.05522773265838623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,4096,3584,0.04993706544240316
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,4096,2560,0.037205334504445395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,4096,3072,0.04345173438390096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,4096,1536,0.02481493353843689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,4096,2048,0.031027199824651082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,4096,1024,0.018705066045125326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,4096,512,0.01269760032494863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,4096,768,0.01576959987481435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,4096,64,0.006075733403364817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,4096,256,0.00962559978167216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,4096,128,0.0076799998680750535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,4096,32,0.005358933409055074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3584,6144,0.07942826747894287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3584,10240,0.13653333981831867
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3584,12288,0.16042666435241698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3584,7168,0.10042026837666829
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3584,8192,0.11253759860992432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3584,4096,0.05485226710637411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3584,3584,0.05355519851048788
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3584,16384,0.22254932721455895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3584,5120,0.06734506289164224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,4096,65536,0.8085162480672201
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3584,2560,0.03682986497879028
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3584,3072,0.042734932899475095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3584,2048,0.030583467086156207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3584,1536,0.024644267559051514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3584,1024,0.01853440006573995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3584,768,0.015564800302187601
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3584,512,0.01225386659304301
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3584,256,0.009045333663622538
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3584,128,0.0073728000124295544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3584,32,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3584,64,0.006075733403364817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3072,6144,0.07232853571573893
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3072,7168,0.08151040077209473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3072,8192,0.08823466300964355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3072,10240,0.10577920277913411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3072,12288,0.12503039836883545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3072,16384,0.1602901299794515
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3072,5120,0.06676479975382486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3072,4096,0.05461333195368449
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3584,65536,0.7069695790608723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3072,1024,0.018329600493113198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3072,2560,0.03648853302001953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3072,3072,0.042632532119750974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3072,3584,0.048640000820159915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3072,1536,0.024405332406361897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3072,2048,0.030583467086156207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3072,768,0.01525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3072,512,0.011878400047620138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3072,32,0.005017599960168203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3072,256,0.00890880028406779
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3072,128,0.007167999943097432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3072,64,0.00576853354771932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2560,12288,0.12004693349202473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2560,10240,0.10489173730214436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2560,8192,0.08837119738260904
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2560,6144,0.07065599759419759
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2560,7168,0.08069120248158773
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,3072,65536,0.5537450790405274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2560,16384,0.1521663983662923
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2560,5120,0.06655999819437662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2560,4096,0.05444266796112061
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2560,3584,0.04836693207422892
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2560,2560,0.03635199864705403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2560,3072,0.04242773453394572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2560,2048,0.030242133140563964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2560,1536,0.024234666426976522
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2560,1024,0.018090667327245076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2560,768,0.014984533190727234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2560,512,0.01225386659304301
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2560,256,0.008806399504343669
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2560,128,0.007099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2560,64,0.0056320001681645715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2560,32,0.005154133339722951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2560,65536,0.48899412155151367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2048,12288,0.10168320337931316
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2048,7168,0.057275732358296715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2048,6144,0.05113173325856527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2048,8192,0.07584426403045655
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2048,10240,0.09093120098114013
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2048,16384,0.1313109318415324
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2048,1536,0.017373865842819212
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2048,2048,0.021026132504145305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2048,4096,0.03979946772257487
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2048,3584,0.03601066668828328
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2048,3072,0.028501333793004353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2048,2560,0.025770666201909383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2048,5120,0.051643733183542886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2048,512,0.008874666690826417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2048,768,0.010922666390736897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2048,256,0.006758399804433187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2048,128,0.0056320001681645715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2048,64,0.006007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2048,1024,0.012834133704503379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2048,32,0.00477866679430008
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1536,7168,0.04215466578801473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,2048,65536,0.4343125343322754
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1536,8192,0.046728531519571945
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1536,10240,0.05481813351313273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1536,12288,0.06184959808985392
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1536,6144,0.03822933435440064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1536,16384,0.08229546546936035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1536,5120,0.03379199902216594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1536,4096,0.029525333642959596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1536,3584,0.02757973273595174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1536,3072,0.024268800020217897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1536,2560,0.02109439969062805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1536,1536,0.014745600024859109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1536,2048,0.01781760056813558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1536,1024,0.011195733149846395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1536,768,0.009693866968154908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1536,256,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1536,512,0.007987200220425924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1536,128,0.0053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1536,64,0.005120000243186951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1536,32,0.004198400179545084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1024,16384,0.060108800729115806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1536,65536,0.3796991984049479
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1024,8192,0.03635199864705403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1024,7168,0.03283626635869344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1024,10240,0.04235946734746297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1024,12288,0.04870826800664266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1024,6144,0.029491200049718218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1024,4096,0.023176532983779908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1024,3072,0.020070399840672812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1024,3584,0.021606399615605672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1024,5120,0.026385066906611125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1024,2048,0.015291733543078103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1024,2560,0.018705066045125326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1024,1536,0.01276586651802063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1024,768,0.008635733524958293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1024,1024,0.010069333513577779
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1024,512,0.0076117331782976794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1024,128,0.0052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1024,64,0.004266666869322458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1024,256,0.005836800237496694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1024,65536,0.21227520306905112
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,1024,32,0.003583999971548716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,768,8192,0.03099306623140971
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,768,10240,0.03594239950180054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,768,12288,0.03822933435440064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,768,16384,0.04734293222427368
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,768,6144,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,768,7168,0.028501333793004353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,768,5120,0.022971733411153158
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,768,3072,0.016622933745384216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,768,4096,0.02044586737950643
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,768,3584,0.019797333081563315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,768,2560,0.01518933375676473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,768,2048,0.012731732924779257
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,768,1536,0.011025066177050274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,768,1024,0.008840533097585042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,768,768,0.007816533247629803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,768,128,0.004881066580613455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,768,256,0.0054613331953684485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,768,512,0.0065194666385650635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,768,64,0.0040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,768,32,0.0034815999368826545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,768,65536,0.15544320742289225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,512,8192,0.022425599892934165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,512,7168,0.020718934138615926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,512,10240,0.025668267409006757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,512,12288,0.029422932863235475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,512,16384,0.03556693394978841
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,512,6144,0.019182932376861573
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,512,5120,0.017851734161376955
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,512,2560,0.013141333063443502
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,512,3072,0.014847999811172486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,512,4096,0.016110933820406594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,512,3584,0.017237333456675212
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,512,2048,0.01129813293615977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,512,1536,0.00955733358860016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,512,1024,0.007918933530648549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,512,768,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,512,512,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,512,256,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,512,64,0.00365226666132609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,512,128,0.004437333345413208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,512,32,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,512,65536,0.10697387059529621
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,256,8192,0.016793600718180337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,256,7168,0.014574933052062988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,256,10240,0.017851734161376955
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,256,12288,0.019012266397476198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,256,16384,0.023859200874964397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,256,6144,0.013516799608866373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,256,5120,0.012800000111262002
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,256,3584,0.011776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,256,4096,0.011229866743087768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,256,3072,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,256,2560,0.010513066252072652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,256,2048,0.009250133236249288
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,256,1024,0.007884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,256,1536,0.00962559978167216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,256,65536,0.07014400164286295
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,256,512,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,256,256,0.005188266436258951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,256,64,0.00314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,256,128,0.00365226666132609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,256,768,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,256,32,0.00314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,128,8192,0.011844266454378765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,128,7168,0.01129813293615977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,128,6144,0.010683733224868774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,128,12288,0.015223466356595359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,128,10240,0.013004799683888754
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,128,16384,0.017749333381652833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,128,5120,0.010069333513577779
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,128,4096,0.009181867043177288
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,128,2560,0.008226133386294047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,128,3584,0.008806399504343669
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,128,1536,0.007441066702206929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,128,3072,0.008533333738644917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,128,2048,0.007782400151093801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,128,768,0.006894933183987935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,128,512,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,128,256,0.004369066655635833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,128,1024,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,128,128,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,128,32,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,128,64,0.0033791999022165934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,128,65536,0.05833386580149332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,64,7168,0.009693866968154908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,64,8192,0.010308266679445902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,64,6144,0.009181867043177288
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,64,12288,0.01153706709543864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,64,16384,0.013312000036239623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,64,10240,0.011571199695269267
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,64,5120,0.009045333663622538
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,64,3584,0.008191999793052674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,64,4096,0.008430932958920796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,64,2560,0.007748266557852428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,64,3072,0.00795306662718455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,64,2048,0.007577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,64,1536,0.006826666494210561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,64,1024,0.006485333542029063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,64,768,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,64,512,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,64,256,0.004266666869322458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,64,128,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,64,64,0.004095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,64,65536,0.045431466897328694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,64,32,0.004198400179545084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,32,7168,0.008703999718030294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,32,6144,0.008260266482830047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,32,8192,0.00897706647713979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,32,10240,0.009693866968154908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,32,12288,0.01027413308620453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,32,16384,0.01181013286113739
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,32,5120,0.007918933530648549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,32,4096,0.007884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,32,3584,0.007406933108965556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,32,2560,0.007406933108965556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,32,2048,0.007099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,32,1536,0.006724266707897187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,32,3072,0.00730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,32,65536,0.040516265233357746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,32,1024,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,32,512,0.005870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,32,768,0.006724266707897187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,32,64,0.0040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,32,256,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,32,128,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,384,32,32,0.004130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,65536,4096,0.526199467976888
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,65536,3584,0.46499840418497723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,65536,5120,0.6537216186523438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,65536,12288,1.6224938710530599
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,65536,6144,0.7682389577229818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,65536,3072,0.4033535957336426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,65536,7168,0.9096192042032877
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,65536,2048,0.2789717356363932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,65536,1024,0.16302080154418946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,65536,16384,2.210815938313802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,65536,768,0.1287168025970459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,65536,8192,1.0333183924357097
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,65536,256,0.0626688003540039
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,65536,512,0.09410560131072998
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,65536,1536,0.21719039281209312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,65536,128,0.05447680155436198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,65536,2560,0.3417429288228353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,65536,32,0.031880533695220946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,65536,10240,1.2829354604085288
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,65536,64,0.04276906649271647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,16384,10240,0.3671039899190267
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,16384,5120,0.25545387268066405
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,16384,6144,0.30167039235432946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,16384,7168,0.35092480977376306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,16384,16384,0.5484544118245442
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,16384,8192,0.401305611928304
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,16384,4096,0.20650666554768882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,16384,12288,0.41949866612752273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,16384,3584,0.18298880259195965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,16384,1536,0.09011200269063314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,16384,2048,0.11520000298817952
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,16384,1024,0.06260053316752115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,16384,3072,0.15858346621195477
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,16384,2560,0.13475839296976727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,16384,512,0.03700053294499715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,16384,768,0.05184853474299113
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,16384,256,0.023756800095240276
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,16384,32,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,16384,64,0.015086932977040609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,16384,128,0.01986560026804606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,12288,6144,0.1530197302500407
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,12288,7168,0.1779029369354248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,12288,4096,0.10451626777648926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,12288,5120,0.12861440181732178
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,12288,8192,0.20019200642903648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,12288,10240,0.25234773953755696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,12288,3584,0.09270613193511963
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,12288,12288,0.2971989313761393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,12288,3072,0.08116906483968099
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,12288,16384,0.3963562647501627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,12288,2560,0.07403519948323568
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,12288,2048,0.056968533992767335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,12288,1536,0.050312534968058265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,12288,768,0.027818665901819868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,12288,1024,0.04212053219477336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,12288,64,0.008772266904513042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,12288,512,0.019899733861287437
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,12288,256,0.013209600249926248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,12288,128,0.010752000411351522
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,12288,32,0.007577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,10240,7168,0.17722026507059735
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,16384,65536,2.300177001953125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,10240,16384,0.33075199127197263
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,10240,8192,0.1998165289560954
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,10240,6144,0.15206400553385419
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,10240,10240,0.24678400357564292
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,10240,12288,0.2957994778951009
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,10240,5120,0.12772693634033203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,10240,4096,0.10379947026570638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,10240,3072,0.08017919858296713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,10240,1024,0.03368959824244182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,10240,1536,0.04894719918568929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,10240,2560,0.067686398824056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,10240,512,0.018943999210993448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,10240,2048,0.056081068515777585
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,10240,768,0.025838933388392132
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,10240,3584,0.09185280005137125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,10240,64,0.008601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,10240,256,0.01276586651802063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,10240,128,0.010547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,10240,32,0.007406933108965556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,8192,12288,0.22278827031453452
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,8192,10240,0.19377493858337402
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,8192,7168,0.14414505958557128
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,8192,8192,0.15810559590657552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,8192,5120,0.10799787044525147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,8192,6144,0.12526933352152508
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,8192,16384,0.28808533350626625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,12288,65536,1.5870293935139974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,8192,3072,0.07929173310597738
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,8192,2560,0.06775466601053873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,8192,3584,0.08461653391520182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,8192,4096,0.09168213208516439
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,8192,2048,0.05533013343811035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,8192,1536,0.04532906611760457
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,8192,1024,0.03099306623140971
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,8192,512,0.01884160041809082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,8192,256,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,8192,768,0.024985599517822265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,8192,128,0.01027413308620453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,8192,64,0.008430932958920796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,8192,32,0.00730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,7168,7168,0.1396394729614258
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,7168,12288,0.22063786188761392
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,8192,65536,1.0900821685791016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,10240,65536,1.3138261159261069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,7168,8192,0.15472639401753743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,7168,10240,0.18896212577819824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,7168,6144,0.12161706288655598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,7168,16384,0.2852863947550456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,7168,5120,0.10687146981557209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,7168,4096,0.09195520083109537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,7168,2048,0.05519359906514486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,7168,3584,0.08243199984232584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,7168,3072,0.0736255963643392
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,7168,2560,0.06717440287272135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,7168,1024,0.030822400252024335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,7168,1536,0.043383467197418216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,7168,768,0.02481493353843689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,7168,512,0.018568533658981323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,7168,128,0.010308266679445902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,7168,256,0.012117333213488261
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,7168,64,0.008362666765848795
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,7168,32,0.007167999943097432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,6144,6144,0.0903167963027954
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,6144,12288,0.1626794656117757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,6144,16384,0.22196906407674155
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,6144,7168,0.10147840181986492
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,6144,8192,0.1137664000193278
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,6144,5120,0.08383146921793619
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,6144,10240,0.1377962589263916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,6144,3584,0.054579198360443115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,6144,4096,0.05775359869003296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,6144,2048,0.03232426643371582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,6144,1024,0.018875734011332194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,6144,768,0.015735466281572977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,6144,3072,0.04727466503779094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,6144,2560,0.040447998046875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,6144,1536,0.025497599442799883
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,6144,512,0.01276586651802063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,6144,128,0.00795306662718455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,6144,256,0.00962559978167216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,7168,65536,0.9425578435262045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,6144,64,0.006109866499900818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,6144,32,0.005188266436258951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,5120,16384,0.18824532826741536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,5120,10240,0.13680639266967773
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,5120,12288,0.1715541362762451
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,5120,6144,0.08028159936269125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,5120,8192,0.11301546891530354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,5120,7168,0.10055680274963379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,5120,5120,0.06942719618479411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,5120,4096,0.055569068590799964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,5120,3584,0.050995198885599766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,5120,1536,0.024644267559051514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,5120,768,0.01546239952246348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,5120,1024,0.01867093245188395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,5120,3072,0.04430506626764934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,5120,2048,0.030719999472300214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,5120,2560,0.0370688001314799
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,5120,512,0.012390399972597759
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,6144,65536,0.7661567687988281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,5120,128,0.0076799998680750535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,5120,64,0.006041599810123444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,5120,32,0.005085866649945577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,5120,256,0.00890880028406779
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,4096,16384,0.15482880274454752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,4096,10240,0.10495999654134114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,4096,6144,0.07260159651438394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,4096,12288,0.1215488036473592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,4096,7168,0.07966720263163249
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,4096,8192,0.08823466300964355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,4096,5120,0.06707200209299723
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,4096,4096,0.05481813351313273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,5120,65536,0.6289749145507812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,4096,3584,0.04877653519312541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,4096,1536,0.024405332406361897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,4096,2048,0.03031040032704671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,4096,3072,0.042871467272440594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,4096,2560,0.03648853302001953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,4096,1024,0.018397865692774455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,4096,256,0.008840533097585042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,4096,768,0.015223466356595359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,4096,128,0.007202133536338806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,4096,64,0.005666133264700572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,4096,32,0.005358933409055074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,4096,512,0.012492799758911132
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3584,16384,0.15158613522847492
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3584,6144,0.0710314671198527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3584,8192,0.08755199909210205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3584,7168,0.07895039717356364
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,4096,65536,0.5480789184570313
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3584,12288,0.11806720097859699
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3584,10240,0.11154773235321044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3584,5120,0.06676479975382486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3584,4096,0.054408534367879235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3584,2048,0.030242133140563964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3584,3584,0.048469332853953044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3584,2560,0.03638613224029541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3584,3072,0.04235946734746297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3584,1536,0.024200532833735147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3584,1024,0.01812480092048645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3584,512,0.01225386659304301
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3584,256,0.008840533097585042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3584,128,0.007133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3584,64,0.0056320001681645715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3584,32,0.005017599960168203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3584,768,0.014984533190727234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3584,65536,0.48841387430826827
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3072,6144,0.05304319858551025
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3072,7168,0.07348906993865967
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3072,8192,0.07621973355611165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3072,10240,0.09130667050679525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3072,12288,0.10349226792653402
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3072,5120,0.048264535268147786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3072,16384,0.12424533367156983
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3072,2560,0.02539520064989726
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3072,3584,0.03768320083618164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3072,3072,0.0291157325108846
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3072,4096,0.04143786827723185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3072,2048,0.02167466680208842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3072,1024,0.013209600249926248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3072,1536,0.01781760056813558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3072,768,0.011127466956774395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3072,512,0.008942932883898417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3072,256,0.006792533397674561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3072,64,0.006007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3072,128,0.005700266857941946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3072,32,0.00477866679430008
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,3072,65536,0.401476256052653
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2560,7168,0.05027840137481689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2560,6144,0.04631893237431844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2560,8192,0.05570559899012247
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2560,5120,0.04939093192418416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2560,10240,0.07533226807912191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2560,12288,0.08751786549886068
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2560,16384,0.11202560265858967
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2560,4096,0.035259731610616046
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2560,3584,0.03068586587905884
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2560,3072,0.026862933238347368
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2560,2560,0.023995733261108397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2560,2048,0.019899733861287437
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2560,1536,0.016179200013478598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2560,1024,0.01269760032494863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2560,768,0.010513066252072652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2560,512,0.00846506655216217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2560,256,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2560,128,0.0055637334783871974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2560,64,0.0052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2560,32,0.004300799965858459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2560,65536,0.38809601465861
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2048,8192,0.05290666818618774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2048,16384,0.08137386639912923
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2048,6144,0.04478293259938558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2048,7168,0.048913065592447916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2048,10240,0.05570559899012247
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2048,12288,0.06287360191345215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2048,5120,0.035225598017374675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2048,4096,0.029149866104125975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2048,3584,0.027409066756566364
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2048,3072,0.024166399240493776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2048,2560,0.021026132504145305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2048,2048,0.017681066195170084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2048,768,0.00942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2048,1536,0.014574933052062988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2048,1024,0.011264000336329143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2048,512,0.007714133461316426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2048,32,0.003925333420435587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2048,64,0.004846933484077454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2048,256,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2048,128,0.0053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,2048,65536,0.3064490636189779
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1536,16384,0.062395731608072914
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1536,12288,0.050995198885599766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1536,10240,0.0446122686068217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1536,8192,0.037307735284169516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1536,6144,0.030583467086156207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1536,7168,0.034406399726867674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1536,5120,0.026897066831588747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1536,3072,0.020138667027155558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1536,3584,0.021811199188232423
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1536,4096,0.023449599742889404
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1536,2560,0.018978132804234823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1536,1536,0.012970667084058127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1536,2048,0.01525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1536,768,0.009284266829490661
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1536,1024,0.010137599706649781
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1536,512,0.0076117331782976794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1536,256,0.005870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1536,128,0.0052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1536,65536,0.21087573369344076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1536,64,0.004266666869322458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1536,32,0.0035157332817713416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1024,16384,0.048537600040435794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1024,12288,0.03846826553344727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1024,10240,0.03621546824773152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1024,8192,0.03078826665878296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1024,7168,0.028501333793004353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1024,5120,0.02259626587231954
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1024,6144,0.025634133815765382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1024,4096,0.020275199413299562
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1024,3584,0.01952426632245382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1024,3072,0.016145066420237223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1024,2560,0.01460906664530436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1024,2048,0.012390399972597759
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1024,1024,0.008567466338475545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1024,1536,0.010342400272687275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1024,768,0.0075434664885203045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1024,65536,0.1512106736501058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1024,128,0.004710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1024,64,0.0038912000755469004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1024,32,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1024,512,0.006451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,1024,256,0.005358933409055074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,768,12288,0.031300266583760575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,768,6144,0.01914880077044169
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,768,7168,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,768,16384,0.03689813216527303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,768,8192,0.022528000672658286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,768,10240,0.02645333409309387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,768,5120,0.017783466974894205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,768,4096,0.01621333360671997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,768,1536,0.009591466188430786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,768,2560,0.013277866442998252
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,768,2048,0.011332266529401143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,768,3072,0.015121066570281982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,768,3584,0.02321066657702128
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,768,1024,0.007918933530648549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,768,65536,0.11219627062479656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,768,768,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,768,512,0.006178133189678192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,768,256,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,768,128,0.0046079998215039575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,768,64,0.0036864000062147772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,768,32,0.0032768001159032187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,512,12288,0.022801067431767783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,512,16384,0.03078826665878296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,512,7168,0.019285333156585694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,512,8192,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,512,6144,0.017749333381652833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,512,10240,0.023381332556406655
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,512,5120,0.016247466206550598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,512,4096,0.013892267147699991
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,512,3584,0.012970667084058127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,512,1536,0.009693866968154908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,512,2560,0.011229866743087768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,512,3072,0.011980799833933513
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,512,2048,0.01027413308620453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,512,768,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,512,1024,0.007884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,512,65536,0.08468480110168457
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,512,512,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,512,256,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,512,128,0.004232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,512,64,0.008294400076071422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,512,32,0.0032085334261258446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,256,10240,0.01546239952246348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,256,7168,0.012902399897575379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,256,8192,0.013823999961217245
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,256,6144,0.011912533640861511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,256,12288,0.018090667327245076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,256,16384,0.021128533283869426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,256,5120,0.011059199770291645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,256,3584,0.009830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,256,3072,0.009386666615804036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,256,2560,0.008942932883898417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,256,1024,0.007782400151093801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,256,4096,0.010137599706649781
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,256,65536,0.06106453339258829
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,256,1536,0.007884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,256,2048,0.008430932958920796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,256,768,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,256,512,0.006041599810123444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,256,256,0.0044714664419492085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,256,128,0.0034815999368826545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,256,64,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,256,32,0.0030037333567937215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,128,16384,0.013926399747530618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,128,8192,0.01027413308620453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,128,5120,0.008942932883898417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,128,6144,0.009250133236249288
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,128,7168,0.009796266754468281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,128,10240,0.011059199770291645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,128,12288,0.012014933427174886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,128,4096,0.008260266482830047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,128,65536,0.04174506664276123
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,128,1024,0.0065536002318064375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,128,1536,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,128,3584,0.007884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,128,3072,0.007645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,128,2048,0.006963199873765309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,128,2560,0.007406933108965556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,128,256,0.004334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,128,128,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,128,64,0.0040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,128,32,0.004130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,128,768,0.006894933183987935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,128,512,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,64,16384,0.010717866818110149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,64,8192,0.008567466338475545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,64,7168,0.008157866696516674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,64,10240,0.008874666690826417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,64,12288,0.009386666615804036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,64,5120,0.007577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,64,4096,0.007202133536338806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,64,6144,0.007850666840871174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,64,65536,0.03269973397254944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,64,3584,0.007748266557852428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,64,2048,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,64,2560,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,64,3072,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,64,1024,0.00628053347269694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,64,1536,0.006621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,64,768,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,64,512,0.006041599810123444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,64,256,0.004198400179545084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,64,128,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,64,64,0.004095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,64,32,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,32,10240,0.008499200145403545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,32,8192,0.008055466910203297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,32,6144,0.00747519979874293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,32,7168,0.007714133461316426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,32,16384,0.010035199920336406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,32,12288,0.009011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,32,5120,0.007236266632874806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,32,4096,0.006929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,32,2560,0.007236266632874806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,32,3072,0.006792533397674561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,32,65536,0.031573333342870075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,32,3584,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,32,2048,0.006451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,32,1536,0.006485333542029063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,32,1024,0.006109866499900818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,32,768,0.006690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,32,512,0.005905066430568695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,32,256,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,32,64,0.0040277334551016486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,32,128,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,256,32,32,0.004130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,65536,4096,0.5086890538533528
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,65536,3584,0.4505258560180664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,65536,12288,1.5680511474609375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,65536,5120,0.6311253229777019
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,65536,3072,0.39069013595581054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,65536,6144,0.7556437174479167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,65536,7168,0.883575439453125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,65536,1536,0.210534397761027
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,65536,8192,1.0036223729451499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,65536,2560,0.33239040374755857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,65536,16384,2.148420206705729
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,65536,1024,0.15841280619303386
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,65536,768,0.12513279914855957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,65536,128,0.051438931624094644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,65536,256,0.05870933135350546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,65536,2048,0.27054080963134763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,65536,64,0.04031146764755249
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,65536,512,0.09239892959594727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,65536,32,0.030446932713190718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,16384,7168,0.2846720059712728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,65536,10240,1.2768255869547525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,16384,8192,0.3205120086669922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,16384,5120,0.25309866269429526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,16384,16384,0.5199530601501465
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,16384,3584,0.1820672035217285
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,16384,6144,0.30057814915974934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,16384,10240,0.3595946629842123
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,16384,12288,0.4229119936625163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,16384,4096,0.2074282646179199
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,16384,3072,0.158515199025472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,16384,2560,0.13397332827250164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,16384,2048,0.11001173655192058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,16384,1536,0.08888320128122965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,16384,1024,0.06242986520131429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,16384,768,0.050722134113311765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,16384,512,0.03614720106124878
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,16384,256,0.026760532458623247
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,16384,64,0.010376532872517902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,16384,128,0.018875734011332194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,16384,32,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,12288,10240,0.24808106422424317
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,12288,8192,0.20066986083984376
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,12288,7168,0.1762645403544108
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,12288,4096,0.10417493184407552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,12288,5120,0.1286826690038045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,12288,3584,0.09260373115539551
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,12288,6144,0.1524735927581787
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,12288,12288,0.2954239845275879
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,12288,2560,0.06833493709564209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,12288,3072,0.08038400014241537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,12288,2048,0.05642240047454834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,12288,16384,0.3919871966044108
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,12288,1536,0.05051733255386352
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,12288,768,0.02754559914271037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,12288,512,0.01911466717720032
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,12288,1024,0.03553280035654704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,12288,256,0.013243732849756875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,12288,32,0.007338666419188182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,12288,128,0.010820266604423524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,12288,64,0.008499200145403545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,10240,10240,0.2605397383371989
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,10240,12288,0.2506410598754883
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,10240,5120,0.1279317299524943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,10240,8192,0.2122410615285238
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,10240,7168,0.1900885264078776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,10240,6144,0.1515178680419922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,10240,16384,0.3221503893534342
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,10240,4096,0.10369706948598226
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,10240,3584,0.09168213208516439
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,10240,3072,0.07983787059783935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,10240,2560,0.06772053241729736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,10240,1536,0.04389546712239583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,10240,256,0.012663466731707254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,10240,2048,0.055637331803639736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,10240,512,0.019182932376861573
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,10240,128,0.010376532872517902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,10240,768,0.025565866629282636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,10240,1024,0.03174399932225545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,10240,64,0.008703999718030294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,10240,32,0.007236266632874806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,16384,65536,2.1343231201171875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,10240,65536,1.286963144938151
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,8192,8192,0.15476053555806477
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,8192,12288,0.21910187403361
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,8192,10240,0.18855253855387372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,8192,7168,0.14213120142618815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,12288,65536,1.5370240529378256
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,8192,6144,0.12356266975402833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,8192,16384,0.2837162653605143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,8192,5120,0.10734933217366535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,8192,4096,0.06417066653569539
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,8192,1536,0.0291157325108846
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,8192,2560,0.0443391998608907
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,8192,1024,0.020172800620396933
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,8192,3584,0.05932373205820719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,8192,2048,0.03508906761805217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,8192,3072,0.05017600059509277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,8192,768,0.018602667252222697
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,8192,512,0.01518933375676473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,8192,256,0.00976213316122691
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,8192,64,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,8192,128,0.00795306662718455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,8192,32,0.007167999943097432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,7168,7168,0.13783040046691894
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,7168,6144,0.13335893948872884
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,8192,65536,1.0728106180826822
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,7168,5120,0.07208960056304932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,7168,8192,0.15261012713114422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,7168,4096,0.05785599946975708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,7168,10240,0.18728960355122884
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,7168,12288,0.21654186248779297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,7168,16384,0.2811562538146973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,7168,3584,0.05577386617660522
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,7168,3072,0.048674134413401286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,7168,2048,0.034065067768096924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,7168,1536,0.026897066831588747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,7168,2560,0.04181333382924397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,7168,1024,0.019421867529551187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,7168,512,0.014062933127085366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,7168,768,0.01621333360671997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,7168,256,0.009489066402117411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,7168,128,0.007850666840871174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,7168,64,0.006178133189678192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,7168,32,0.005120000243186951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,6144,8192,0.11342506408691407
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,6144,10240,0.1377621332804362
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,6144,16384,0.21067093213399252
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,6144,12288,0.16100692749023438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,6144,4096,0.057241598765055336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,6144,6144,0.09198933442433675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,6144,5120,0.06932480335235595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,6144,7168,0.10250240166982014
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,6144,3584,0.05263359944025675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,6144,768,0.015871999661127727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,6144,2560,0.038809601465861
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,6144,1536,0.025224532683690386
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,6144,3072,0.04580693244934082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,6144,1024,0.018636800845464072
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,6144,2048,0.0315391997496287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,6144,256,0.009045333663622538
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,6144,128,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,6144,512,0.013346133629480996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,6144,64,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,7168,65536,0.9152170817057291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,6144,32,0.005120000243186951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,5120,12288,0.16076800028483074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,5120,16384,0.1878357410430908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,5120,10240,0.1362602710723877
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,5120,7168,0.10072746276855468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,5120,6144,0.07973546981811523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,5120,8192,0.11209386984507244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,5120,4096,0.05505706469217936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,5120,5120,0.06833493709564209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,5120,3584,0.05021013418833414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,5120,1536,0.024473599592844644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,5120,2048,0.03051519989967346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,5120,2560,0.037444265683492024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,5120,3072,0.043315200010935466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,5120,1024,0.01843199928601583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,5120,768,0.015530666708946228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,6144,65536,0.7342762629191081
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,5120,512,0.01256106694539388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,5120,256,0.008874666690826417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,5120,32,0.005188266436258951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,5120,128,0.007202133536338806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,5120,64,0.00576853354771932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,4096,8192,0.08772266705830892
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,4096,12288,0.12209493319193523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,4096,7168,0.07932586669921875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,4096,6144,0.05478399991989136
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,4096,10240,0.10451626777648926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,4096,16384,0.15223466555277507
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,4096,5120,0.06686720053354898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,4096,3072,0.029798400402069092
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,4096,3584,0.035225598017374675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,5120,65536,0.6171989440917969
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,4096,2560,0.026555732885996504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,4096,4096,0.05451093514760336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,4096,2048,0.022937599817911783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,4096,1536,0.018602667252222697
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,4096,1024,0.013619200388590495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,4096,768,0.01153706709543864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,4096,512,0.009113599856694538
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,4096,256,0.006792533397674561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,4096,64,0.006041599810123444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,4096,128,0.005870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,4096,32,0.004744533201058706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,4096,65536,0.5490346908569336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3584,6144,0.06143999894460043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3584,7168,0.0702122688293457
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3584,8192,0.07796053091684976
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3584,10240,0.0939690669377645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3584,5120,0.04519253174463908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3584,12288,0.10929493109385173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3584,16384,0.1422335942586263
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3584,4096,0.03590826590855916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3584,2048,0.01986560026804606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3584,3584,0.031982932488123575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3584,2560,0.024029866854349772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3584,3072,0.027511467536290485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3584,1536,0.01641813317934672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3584,1024,0.012663466731707254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3584,768,0.0106495996316274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3584,512,0.008703999718030294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3584,256,0.006485333542029063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3584,128,0.0056320001681645715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3584,64,0.005666133264700572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3584,32,0.0045056000351905824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3072,12288,0.0879957358042399
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3072,10240,0.07683413028717041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3072,16384,0.12410879929860433
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3072,6144,0.049459199110666915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3072,7168,0.05560319821039835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3072,8192,0.06137173175811768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3584,65536,0.4781397183736165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3072,5120,0.040174933274586995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3072,4096,0.032051199674606325
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3072,3584,0.02945706645647685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3072,2048,0.01802240014076233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3072,1536,0.014813866217931113
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3072,2560,0.021606399615605672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3072,3072,0.025326933463414508
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3072,1024,0.011434666315714518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3072,768,0.009898666540781658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3072,512,0.008055466910203297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3072,256,0.006382933259010315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3072,128,0.005393066505591074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3072,32,0.004198400179545084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3072,64,0.005188266436258951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2560,16384,0.093013334274292
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,3072,65536,0.3894613265991211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2560,12288,0.07321600119272867
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2560,6144,0.04440746704737346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2560,7168,0.048469332853953044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2560,8192,0.05314559936523437
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2560,10240,0.06287360191345215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2560,5120,0.03839999834696452
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2560,4096,0.030958932638168336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2560,3072,0.024200532833735147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2560,3584,0.02792106668154399
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2560,2048,0.017749333381652833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2560,1536,0.014574933052062988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2560,2560,0.021026132504145305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2560,1024,0.011229866743087768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2560,512,0.007748266557852428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2560,768,0.00962559978167216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2560,128,0.005120000243186951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2560,64,0.004744533201058706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2560,32,0.003822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2560,256,0.006178133189678192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2560,65536,0.32320852279663087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2048,6144,0.03696639935175578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2048,8192,0.051507198810577394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2048,7168,0.04143786827723185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2048,12288,0.0552618662516276
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2048,10240,0.048230401674906415
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2048,5120,0.03300693432490031
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2048,16384,0.07871146996816
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2048,4096,0.027613866329193115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2048,3584,0.024200532833735147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2048,3072,0.020957867304484047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2048,2560,0.01935360034306844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2048,2048,0.01539413332939148
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2048,1024,0.010240000486373902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2048,1536,0.013243732849756875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2048,512,0.007645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2048,768,0.00897706647713979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2048,256,0.005973333120346069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2048,128,0.005256533126036326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2048,64,0.004300799965858459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2048,32,0.0039594667653242745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,2048,65536,0.29351253509521485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1536,7168,0.03297280073165894
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1536,12288,0.05495466788609823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1536,6144,0.0291157325108846
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1536,8192,0.03590826590855916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1536,10240,0.04352000157038371
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1536,5120,0.025873066981633504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1536,16384,0.06802773475646973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1536,4096,0.02228906750679016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1536,3072,0.016964266697565712
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1536,2048,0.013141333063443502
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1536,2560,0.018978132804234823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1536,3584,0.01976319948832194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1536,1024,0.008738133311271667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1536,1536,0.011025066177050274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1536,768,0.007918933530648549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1536,512,0.006690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1536,256,0.005529599885145823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1536,128,0.004915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1536,65536,0.23336960474650065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1536,32,0.0034815999368826545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1536,64,0.0045056000351905824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1024,7168,0.025156267484029132
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1024,6144,0.02239146629969279
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1024,12288,0.037307735284169516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1024,8192,0.027170133590698243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1024,10240,0.03246080080668132
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1024,16384,0.04580693244934082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1024,5120,0.020923733711242676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1024,3584,0.017783466974894205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1024,1536,0.009830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1024,3072,0.01546239952246348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1024,2048,0.011468799908955891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1024,2560,0.013653332988421122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1024,1024,0.007918933530648549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1024,4096,0.018602667252222697
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1024,65536,0.14295040766398112
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1024,768,0.007099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1024,512,0.006178133189678192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1024,256,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1024,128,0.0046079998215039575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1024,64,0.0037546666959921518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,1024,32,0.0032768001159032187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,768,16384,0.037102933724721274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,768,8192,0.021504000822703043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,768,12288,0.028637866179148357
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,768,6144,0.018295466899871826
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,768,7168,0.019831466674804687
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,768,10240,0.02474666635195414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,768,5120,0.016110933820406594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,768,4096,0.014233600099881491
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,768,3072,0.01256106694539388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,768,3584,0.013414399822553
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,768,1536,0.009659733374913533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,768,2560,0.01153706709543864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,768,2048,0.011434666315714518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,768,65536,0.10758826732635499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,768,768,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,768,1024,0.007918933530648549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,768,512,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,768,128,0.004334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,768,256,0.005188266436258951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,768,64,0.0035157332817713416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,768,32,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,512,16384,0.02768213351567586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,512,12288,0.021640533208847047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,512,8192,0.016657066345214844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,512,7168,0.014882133404413859
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,512,10240,0.017885865767796834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,512,5120,0.012868266304334006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,512,6144,0.013823999961217245
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,512,4096,0.01153706709543864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,512,2048,0.009284266829490661
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,512,2560,0.010513066252072652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,512,3072,0.011264000336329143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,512,1536,0.009659733374913533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,512,1024,0.007884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,512,3584,0.011980799833933513
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,512,65536,0.0818176031112671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,512,768,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,512,512,0.006178133189678192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,512,256,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,512,128,0.0037205333511034647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,512,64,0.0031744000812371576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,512,32,0.0030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,256,16384,0.01576959987481435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,256,7168,0.010990933577219645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,256,8192,0.012117333213488261
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,256,10240,0.012663466731707254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,256,6144,0.010547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,256,5120,0.010137599706649781
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,256,12288,0.013619200388590495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,256,4096,0.009284266829490661
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,256,65536,0.05495466788609823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,256,2560,0.008328533172607422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,256,3584,0.008942932883898417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,256,3072,0.008601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,256,2048,0.007816533247629803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,256,1536,0.00747519979874293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,256,1024,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,256,768,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,256,512,0.005973333120346069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,256,256,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,256,128,0.0033791999022165934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,256,64,0.004095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,256,32,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,128,16384,0.01269760032494863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,128,12288,0.011195733149846395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,128,6144,0.008874666690826417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,128,7168,0.009284266829490661
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,128,8192,0.00976213316122691
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,128,5120,0.00846506655216217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,128,10240,0.010444800059000652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,128,65536,0.038126933574676516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,128,4096,0.008396800359090168
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,128,3584,0.007850666840871174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,128,2048,0.006792533397674561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,128,2560,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,128,3072,0.00730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,128,1536,0.006587733328342438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,128,1024,0.00631466656923294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,128,768,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,128,512,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,128,256,0.004300799965858459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,128,128,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,128,64,0.004095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,128,32,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,64,12288,0.008942932883898417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,64,10240,0.008942932883898417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,64,16384,0.009864532947540283
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,64,8192,0.008703999718030294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,64,5120,0.0075434664885203045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,64,6144,0.007782400151093801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,64,7168,0.008328533172607422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,64,65536,0.028398933013280232
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,64,4096,0.0076799998680750535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,64,3584,0.0073728000124295544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,64,2560,0.006929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,64,1024,0.005666133264700572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,64,2048,0.006485333542029063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,64,1536,0.0065194666385650635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,64,3072,0.007167999943097432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,64,768,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,64,512,0.006007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,64,64,0.0040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,64,32,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,64,256,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,64,128,0.0044714664419492085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,32,12288,0.008430932958920796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,32,16384,0.009352533022562663
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,32,10240,0.008567466338475545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,32,7168,0.008123733103275299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,32,8192,0.008396800359090168
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,32,5120,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,32,6144,0.00747519979874293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,32,65536,0.017749333381652833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,32,4096,0.0075434664885203045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,32,3072,0.006894933183987935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,32,3584,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,32,2048,0.006348800162474315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,32,2560,0.006690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,32,1024,0.0053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,32,1536,0.005836800237496694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,32,768,0.006724266707897187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,32,512,0.005870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,32,256,0.004983466863632202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,32,128,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,32,64,0.0040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,192,32,32,0.004130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,65536,6144,0.7510357538859049
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,65536,4096,0.5081770579020183
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,65536,5120,0.6274389266967774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,65536,3584,0.45021867752075195
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,65536,12288,1.521117909749349
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,65536,2048,0.2705066680908203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,65536,1536,0.20974933306376137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,65536,16384,2.093670399983724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,65536,7168,0.8743935902913412
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,65536,1024,0.15735467274983722
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,65536,2560,0.3302058537801107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,65536,8192,0.9896618525187174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,65536,3072,0.3894954681396484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,65536,768,0.12212906678517658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,65536,256,0.05700266758600871
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,65536,128,0.0495957334836324
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,65536,10240,1.2706474304199218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,65536,512,0.09134079615275065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,65536,64,0.037717334429423016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,65536,32,0.03174399932225545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,16384,7168,0.2800298690795898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,16384,10240,0.39304533004760744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,16384,6144,0.30030508041381837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,16384,5120,0.2521770636240641
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,16384,8192,0.31675732930501305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,16384,16384,0.5171541213989258
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,16384,4096,0.2041855971018473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,16384,3584,0.18189652760823566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,16384,1024,0.06133759816487631
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,16384,1536,0.08826879660288492
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,16384,768,0.0504149317741394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,16384,12288,0.45854721069335935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,16384,3072,0.1572864055633545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,16384,2560,0.13230079809824627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,16384,2048,0.11124053001403808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,16384,512,0.03597653309504191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,16384,256,0.017681066195170084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,16384,128,0.013277866442998252
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,16384,64,0.010035199920336406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,16384,32,0.009659733374913533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,12288,8192,0.21517653465270997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,12288,10240,0.2602325280507406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,12288,12288,0.30808747609456383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,12288,3584,0.09202346801757813
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,12288,6144,0.15182506243387858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,12288,5120,0.1285802682240804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,12288,4096,0.10417493184407552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,12288,7168,0.18862080574035645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,12288,1536,0.0495957334836324
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,12288,2048,0.05655893484751383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,12288,16384,0.3920554796854655
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,12288,1024,0.03423573176066081
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,12288,2560,0.06826666990915933
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,12288,768,0.025941334168116253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,12288,3072,0.08014506498972575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,12288,512,0.019592533508936562
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,12288,32,0.00747519979874293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,12288,256,0.013107200463612875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,12288,128,0.010376532872517902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,12288,64,0.00846506655216217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,10240,12288,0.30924800237019856
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,10240,16384,0.3226282755533854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,10240,7168,0.1878357410430908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,10240,6144,0.1660586675008138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,10240,10240,0.2600618680318197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,10240,8192,0.21179733276367188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,10240,5120,0.14400854110717773
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,10240,3584,0.10543786684672038
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,10240,2048,0.057378133138020836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,10240,2560,0.07932586669921875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,10240,4096,0.11622400283813476
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,10240,3072,0.0934229294459025
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,10240,1536,0.04669440189997355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,10240,1024,0.0315391997496287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,10240,128,0.010342400272687275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,10240,64,0.008499200145403545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,10240,512,0.018943999210993448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,10240,256,0.01276586651802063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,10240,768,0.02553173303604126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,10240,32,0.007133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,8192,8192,0.1577642599741618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,10240,65536,1.22511355082194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,8192,7168,0.1392639954884847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,8192,12288,0.2185215950012207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,8192,10240,0.18643627166748047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,16384,65536,2.064349873860677
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,8192,6144,0.12332373460133869
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,8192,16384,0.2821119944254557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,8192,5120,0.07758506933848063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,12288,65536,1.4966784159342448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,8192,4096,0.06283946832021078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,8192,2048,0.03577173153559367
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,8192,3584,0.05833386580149332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,8192,3072,0.05123413403828939
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,8192,1536,0.028910932938257854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,8192,2560,0.04235946734746297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,8192,1024,0.021845332781473794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,8192,64,0.006075733403364817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,8192,512,0.014097066720326743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,8192,128,0.007850666840871174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,8192,768,0.018261333306630455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,8192,256,0.00993280013402303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,8192,32,0.006348800162474315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,7168,6144,0.08471893469492595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,7168,16384,0.28081493377685546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,8192,65536,1.0717183430989583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,7168,7168,0.09717760086059571
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,7168,8192,0.1514154593149821
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,7168,10240,0.18537813822428387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,7168,5120,0.07195306619008382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,7168,12288,0.214630397160848
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,7168,4096,0.06007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,7168,3584,0.05519359906514486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,7168,1536,0.02696533401807149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,7168,3072,0.04795733292897542
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,7168,2560,0.04116479953130086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,7168,2048,0.03389439980189006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,7168,1024,0.020411733786265054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,7168,768,0.016964266697565712
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,7168,256,0.009386666615804036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,7168,512,0.013482667009035745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,7168,128,0.007577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,7168,64,0.00580266664425532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,7168,32,0.005870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,6144,10240,0.1386496067047119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,6144,16384,0.2086911996205648
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,6144,12288,0.16121172904968262
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,6144,7168,0.10147840181986492
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,6144,6144,0.08861013253529867
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,6144,8192,0.11253759860992432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,6144,4096,0.05638826688130697
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,6144,3584,0.05222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,6144,5120,0.06912000179290771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,6144,1536,0.024951465924580894
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,6144,1024,0.018705066045125326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,6144,768,0.016383999586105348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,6144,2048,0.031402667363484696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,6144,2560,0.03870720068613688
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,6144,3072,0.045499734083811444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,6144,512,0.012902399897575379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,6144,256,0.00897706647713979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,6144,128,0.00747519979874293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,6144,32,0.005188266436258951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,6144,64,0.005700266857941946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,7168,65536,0.902348772684733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,5120,12288,0.159880526860555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,5120,10240,0.1374890645345052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,5120,16384,0.18599252700805663
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,5120,7168,0.09983999729156494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,5120,8192,0.11219627062479656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,5120,4096,0.05505706469217936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,5120,5120,0.06782293319702148
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,5120,6144,0.08062293529510497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,5120,3584,0.0497322678565979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,5120,1536,0.024473599592844644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,5120,2048,0.03054933349291484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,5120,2560,0.0421887993812561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,5120,3072,0.04334933360417684
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,6144,65536,0.7282005310058594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,5120,1024,0.01890986760457357
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,5120,768,0.015940266847610473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,5120,512,0.01259519954522451
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,5120,256,0.008840533097585042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,5120,128,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,5120,64,0.0056320001681645715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,5120,32,0.00477866679430008
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,4096,12288,0.1094655990600586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,4096,10240,0.09485653241475424
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,4096,16384,0.14274560610453288
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,4096,6144,0.06174720128377279
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,4096,7168,0.07045119603474935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,4096,8192,0.07840426762898764
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,4096,5120,0.044544001420338944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,4096,4096,0.03682986497879028
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,4096,3584,0.0338261326154073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,5120,65536,0.614297612508138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,4096,1536,0.01713493267695109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,4096,3072,0.028945066531499225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,4096,2048,0.02058239976565043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,4096,2560,0.02539520064989726
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,4096,768,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,4096,1024,0.012936533490816752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,4096,512,0.008806399504343669
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,4096,256,0.0065536002318064375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,4096,128,0.005666133264700572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,4096,32,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,4096,64,0.0052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3584,16384,0.1425066630045573
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3584,10240,0.09390079975128174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3584,7168,0.06980266571044921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3584,6144,0.06113280057907104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3584,12288,0.10926079750061035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3584,8192,0.07720959981282552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,4096,65536,0.5345962524414063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3584,5120,0.04242773453394572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3584,4096,0.034030934174855546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3584,3072,0.026658133665720625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3584,3584,0.03061760067939758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3584,2048,0.018978132804234823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3584,2560,0.023244800170262654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3584,1536,0.015701333681742348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3584,768,0.010478933652242024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3584,1024,0.012083199620246888
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3584,512,0.008499200145403545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3584,128,0.0054613331953684485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3584,64,0.005154133339722951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3584,256,0.006485333542029063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3584,32,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3584,65536,0.47496531804402664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3072,7168,0.05509119828542074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3072,16384,0.10929493109385173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3072,8192,0.0609279990196228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3072,10240,0.07314773400624594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3072,5120,0.039389868577321366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3072,12288,0.08540159861246745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3072,6144,0.04904959996541341
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3072,4096,0.03089066743850708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3072,3072,0.025088000297546386
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3072,2560,0.021401600042978922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3072,3584,0.029047467311223346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3072,1536,0.014813866217931113
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3072,1024,0.011332266529401143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3072,2048,0.01791999936103821
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3072,768,0.009693866968154908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3072,512,0.007884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3072,256,0.006212266782919565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3072,64,0.004710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3072,32,0.003822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3072,128,0.0054613331953684485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,3072,65536,0.3966634750366211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2560,6144,0.04369066556294759
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2560,8192,0.05249706506729126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2560,10240,0.06167893409729004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2560,7168,0.047650134563446044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2560,12288,0.07205546696980794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2560,5120,0.03805866638819377
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2560,16384,0.09164799849192301
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2560,4096,0.030583467086156207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2560,3072,0.024337067206700643
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2560,2048,0.017681066195170084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2560,3584,0.02744320034980774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2560,2560,0.020923733711242676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2560,1536,0.014506666858990987
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2560,1024,0.011229866743087768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2560,768,0.00942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2560,512,0.007748266557852428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2560,256,0.006007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2560,128,0.005085866649945577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2560,64,0.004369066655635833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2560,65536,0.3216383934020996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2560,32,0.003618133316437403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2048,6144,0.03682986497879028
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2048,7168,0.04109653234481812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2048,8192,0.04468053181966146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2048,16384,0.078165332476298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2048,10240,0.05358933210372925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2048,12288,0.061269334952036535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2048,4096,0.027340799570083618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2048,5120,0.03174399932225545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2048,3584,0.02259626587231954
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2048,2560,0.0180565337340037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2048,3072,0.019797333081563315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2048,1024,0.009659733374913533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2048,2048,0.014779733618100485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2048,1536,0.012526933352152506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2048,128,0.005120000243186951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2048,512,0.007202133536338806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2048,64,0.0040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2048,32,0.0034133332471052804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2048,65536,0.26978985468546546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2048,768,0.008430932958920796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,2048,256,0.005836800237496694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1536,10240,0.045943466822306316
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1536,5120,0.02362026572227478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1536,6144,0.027716267108917236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1536,8192,0.03826346794764201
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1536,7168,0.037375998497009275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1536,12288,0.051370668411254886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1536,4096,0.020241065820058187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1536,16384,0.05768533150355021
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1536,3584,0.018636800845464072
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1536,3072,0.016179200013478598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1536,2560,0.014745600024859109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1536,1024,0.008499200145403545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1536,2048,0.01262933313846588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1536,1536,0.010683733224868774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1536,768,0.007645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1536,65536,0.18640213012695311
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1536,512,0.006621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1536,64,0.003925333420435587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1536,128,0.007850666840871174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1536,32,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1536,256,0.005427200098832448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1024,16384,0.043144532044728595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1024,12288,0.03601066668828328
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1024,8192,0.02945706645647685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1024,10240,0.03536213239034017
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1024,7168,0.026692267258961993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1024,6144,0.019182932376861573
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1024,5120,0.017476266622543334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1024,4096,0.01546239952246348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1024,3584,0.014574933052062988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1024,3072,0.014131200313568116
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1024,2560,0.013482667009035745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1024,2048,0.011400533715883891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1024,1536,0.009591466188430786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1024,65536,0.13202773729960124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1024,1024,0.007918933530648549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1024,768,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1024,512,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1024,64,0.0035157332817713416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1024,32,0.0031744000812371576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1024,256,0.005188266436258951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,1024,128,0.004539733131726583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,768,16384,0.03519146839777629
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,768,10240,0.024200532833735147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,768,12288,0.027989333868026732
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,768,6144,0.017271467049916587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,768,5120,0.015701333681742348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,768,8192,0.02058239976565043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,768,7168,0.018807466824849448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,768,4096,0.013994666934013366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,768,3584,0.013175466656684875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,768,3072,0.012390399972597759
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,768,65536,0.10632533232371014
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,768,2560,0.011366400122642516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,768,1536,0.00962559978167216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,768,1024,0.007884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,768,2048,0.011502933502197266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,768,768,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,768,512,0.006178133189678192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,768,256,0.005188266436258951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,768,128,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,768,32,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,768,64,0.0035157332817713416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,512,16384,0.026521599292755126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,512,12288,0.021128533283869426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,512,8192,0.016520532965660095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,512,10240,0.01730560064315796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,512,7168,0.01437013347943624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,512,6144,0.013346133629480996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,512,5120,0.01262933313846588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,512,4096,0.01129813293615977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,512,3072,0.010854400197664897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,512,2560,0.010376532872517902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,512,3584,0.010478933652242024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,512,65536,0.0740010658899943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,512,1536,0.009591466188430786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,512,2048,0.009011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,512,1024,0.007850666840871174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,512,512,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,512,256,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,512,64,0.0034815999368826545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,512,128,0.003583999971548716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,512,768,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,512,32,0.0030720000465710956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,256,16384,0.01546239952246348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,256,10240,0.012356266379356384
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,256,12288,0.013414399822553
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,256,8192,0.011844266454378765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,256,7168,0.010820266604423524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,256,5120,0.00976213316122691
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,256,6144,0.010205866893132527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,256,65536,0.05283840099970499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,256,3584,0.008772266904513042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,256,3072,0.008362666765848795
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,256,2560,0.008089600006739299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,256,1536,0.007167999943097432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,256,2048,0.0076117331782976794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,256,4096,0.009011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,256,1024,0.006894933183987935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,256,768,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,256,256,0.004369066655635833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,256,512,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,256,128,0.0033791999022165934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,256,64,0.004095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,256,32,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,128,12288,0.011025066177050274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,128,16384,0.012800000111262002
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,128,8192,0.009693866968154908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,128,10240,0.010240000486373902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,128,6144,0.008806399504343669
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,128,7168,0.009250133236249288
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,128,5120,0.008328533172607422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,128,65536,0.03659093379974365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,128,4096,0.008362666765848795
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,128,3072,0.00730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,128,2560,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,128,2048,0.006724266707897187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,128,3584,0.007884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,128,1536,0.006587733328342438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,128,1024,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,128,512,0.006075733403364817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,128,768,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,128,256,0.004266666869322458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,128,128,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,128,64,0.004095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,128,32,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,64,16384,0.009659733374913533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,64,12288,0.008874666690826417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,64,6144,0.007748266557852428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,64,8192,0.008806399504343669
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,64,7168,0.008294400076071422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,64,10240,0.00897706647713979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,64,65536,0.021230934063593547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,64,5120,0.007577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,64,1536,0.006417066852251689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,64,4096,0.007748266557852428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,64,3584,0.00730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,64,2560,0.006929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,64,3072,0.007099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,64,2048,0.006417066852251689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,64,1024,0.005666133264700572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,64,768,0.006826666494210561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,64,512,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,64,128,0.0044714664419492085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,64,64,0.004198400179545084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,64,32,0.004130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,64,256,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,32,16384,0.009215999643007915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,32,10240,0.008567466338475545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,32,6144,0.00747519979874293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,32,7168,0.008021333316961924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,32,8192,0.008123733103275299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,32,12288,0.00846506655216217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,32,65536,0.019421867529551187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,32,4096,0.00747519979874293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,32,5120,0.00730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,32,3584,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,32,2560,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,32,3072,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,32,2048,0.00628053347269694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,32,1024,0.0052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,32,1536,0.00576853354771932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,32,768,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,32,256,0.005017599960168203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,32,128,0.004369066655635833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,32,512,0.005870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,32,64,0.0040277334551016486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,160,32,32,0.004130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,65536,6144,0.569378153483073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,65536,8192,0.7350271860758464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,65536,4096,0.3776512145996094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,65536,5120,0.4774911880493164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,65536,3584,0.34208428064982094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,65536,12288,1.0485418955485026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,65536,3072,0.3195562680562337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,65536,1536,0.17868800163269044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,65536,1024,0.12479146321614583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,65536,7168,0.6634837468465169
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,65536,768,0.09871359666188559
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,65536,256,0.0457045316696167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,65536,512,0.07045119603474935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,65536,64,0.024268800020217897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,65536,2048,0.2268160025278727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,65536,128,0.03314346671104431
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,65536,32,0.01935360034306844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,65536,2560,0.2521429379781087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,16384,8192,0.20261546770731606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,65536,10240,0.9287338892618815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,16384,16384,0.36051626205444337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,16384,6144,0.15667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,16384,10240,0.23917226791381835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,16384,7168,0.19271680514017742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,16384,4096,0.10758826732635499
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,16384,12288,0.2789717356363932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,16384,3584,0.09605120023091634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,16384,2048,0.07024640242258708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,16384,5120,0.13765974044799806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,16384,3072,0.08700586954752604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,16384,1536,0.05744640032450358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,16384,2560,0.07342080275217691
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,16384,768,0.03054933349291484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,16384,1024,0.03887786865234375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,16384,32,0.007406933108965556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,16384,512,0.01986560026804606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,16384,128,0.010410666465759277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,16384,64,0.008328533172607422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,16384,256,0.01372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,65536,16384,1.3603499094645182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,12288,16384,0.2678442637125651
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,12288,7168,0.1281706651051839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,12288,8192,0.14021973609924315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,12288,10240,0.17462612787882487
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,12288,12288,0.20862293243408203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,12288,6144,0.12390399773915609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,12288,5120,0.10618879795074462
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,12288,4096,0.08079360326131185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,12288,3584,0.0754688024520874
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,12288,3072,0.066594131787618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,12288,1536,0.03566933472951253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,12288,2560,0.05543253421783447
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,12288,1024,0.024985599517822265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,12288,2048,0.04369066556294759
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,12288,768,0.019182932376861573
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,12288,512,0.014643200238545737
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,12288,256,0.011468799908955891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,12288,128,0.007918933530648549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,12288,64,0.008362666765848795
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,12288,32,0.007236266632874806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,16384,65536,1.3552298227945963
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,10240,6144,0.10994346936543782
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,10240,12288,0.18363733291625978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,10240,8192,0.12069546381632487
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,10240,7168,0.11410773595174153
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,10240,4096,0.0724992036819458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,10240,10240,0.15359999338785807
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,10240,5120,0.08447999954223633
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,10240,16384,0.23412052790323892
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,12288,65536,1.040008544921875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,10240,2560,0.046353065967559816
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,10240,1536,0.029969066381454468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,10240,2048,0.04123306671778361
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,10240,3072,0.05847040017445883
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,10240,3584,0.07007573445638021
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,10240,768,0.017612799008687337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,10240,512,0.013482667009035745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,10240,1024,0.021811199188232423
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,10240,256,0.010137599706649781
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,10240,128,0.0076117331782976794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,10240,32,0.0053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,10240,64,0.00576853354771932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,8192,10240,0.12957013448079427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,8192,16384,0.19384320576985675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,8192,6144,0.08591360251108805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,8192,12288,0.14718292554219564
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,8192,7168,0.09748480319976807
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,8192,8192,0.11359573205312093
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,8192,5120,0.07707306543986002
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,8192,4096,0.06389760176340739
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,8192,1024,0.01925119956334432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,8192,2560,0.04430506626764934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,8192,2048,0.034986666838328045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,8192,3072,0.05236053466796875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,8192,3584,0.0559445341428121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,8192,1536,0.02792106668154399
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,8192,768,0.015530666708946228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,8192,512,0.012936533490816752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,8192,256,0.009215999643007915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,8192,128,0.007167999943097432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,8192,64,0.005734399954477946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,8192,32,0.0053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,7168,7168,0.09018026987711589
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,8192,65536,0.6925653457641602
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,7168,8192,0.10045440196990967
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,7168,10240,0.11789653301239014
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,10240,65536,0.8679765065511068
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,7168,12288,0.13107199668884278
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,7168,16384,0.17216854095458983
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,7168,6144,0.07970133622487387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,7168,5120,0.0746837298075358
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,7168,4096,0.06014293432235718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,7168,3584,0.056866133213043214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,7168,2048,0.03331413269042969
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,7168,2560,0.04154026508331299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,7168,3072,0.0501418670018514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,7168,1536,0.026146133740743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,7168,512,0.01259519954522451
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,7168,768,0.015667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,7168,256,0.008942932883898417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,7168,128,0.007099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,7168,64,0.0055978665749231975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,7168,1024,0.01843199928601583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,7168,32,0.005085866649945577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,6144,7168,0.09041919708251953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,6144,6144,0.0671061356862386
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,7168,65536,0.6118058522542318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,6144,8192,0.09352533022562662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,6144,5120,0.05857280095418295
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,6144,12288,0.12393813133239746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,6144,10240,0.10673493544260662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,6144,3584,0.048913065592447916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,6144,4096,0.04795733292897542
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,6144,16384,0.14953813552856446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,6144,2560,0.028398933013280232
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,6144,1024,0.013960533340771995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,6144,2048,0.026658133665720625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,6144,1536,0.020855466524759926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,6144,3072,0.04334933360417684
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,6144,768,0.01242453356583913
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,6144,512,0.00962559978167216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,6144,128,0.005870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,6144,256,0.006894933183987935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,6144,64,0.006007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,6144,32,0.004744533201058706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,5120,8192,0.07393279870351156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,5120,16384,0.13066240151723224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,5120,10240,0.09113600254058837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,5120,12288,0.10642773310343426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,5120,3584,0.035293865203857425
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,5120,4096,0.04703573385874431
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,5120,7168,0.06929066975911459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,5120,6144,0.0609279990196228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,5120,5120,0.05509119828542074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,5120,2560,0.02611200014750163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,5120,2048,0.0211626668771108
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,5120,768,0.010683733224868774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,5120,3072,0.030958932638168336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,5120,1024,0.012936533490816752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,5120,1536,0.017066667477289833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,5120,512,0.008738133311271667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,6144,65536,0.5359274546305339
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,5120,256,0.006417066852251689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,5120,128,0.0055978665749231975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,5120,64,0.0053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,5120,32,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,4096,10240,0.07208960056304932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,4096,8192,0.06140586535135904
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,4096,16384,0.10659840106964111
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,4096,6144,0.048640000820159915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,4096,12288,0.08731306393941243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,4096,7168,0.054237866401672365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,4096,5120,0.04532906611760457
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,4096,3584,0.03170986572901408
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,4096,4096,0.037307735284169516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,5120,65536,0.4609365463256836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,4096,1536,0.015701333681742348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,4096,3072,0.026385066906611125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,4096,2048,0.01911466717720032
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,4096,2560,0.023108265797297158
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,4096,768,0.010137599706649781
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,4096,1024,0.011878400047620138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,4096,256,0.006178133189678192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,4096,512,0.007918933530648549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,4096,64,0.00481279989083608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,4096,128,0.005256533126036326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,4096,32,0.003925333420435587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3584,12288,0.07488853136698405
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3584,7168,0.050995198885599766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3584,6144,0.04464639822642009
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,4096,65536,0.44342613220214844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3584,16384,0.09878186384836832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3584,8192,0.05587626695632934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3584,10240,0.06669653256734212
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3584,5120,0.04119893312454224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3584,2560,0.02225493391354879
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3584,3584,0.02908160090446472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3584,2048,0.01843199928601583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3584,3072,0.026282666126887004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3584,4096,0.03351893424987793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3584,1536,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3584,1024,0.011366400122642516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3584,768,0.00962559978167216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3584,512,0.0076799998680750535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3584,256,0.006075733403364817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3584,128,0.005154133339722951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3584,64,0.004710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3584,32,0.003788800040880839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3584,65536,0.3335850715637207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3072,7168,0.043656531969706217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3072,16384,0.08062293529510497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3072,8192,0.04689919948577881
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3072,6144,0.039287467797597245
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3072,5120,0.037034666538238524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3072,10240,0.05710506836573283
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3072,12288,0.065774933497111
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3072,3584,0.02696533401807149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3072,4096,0.028569600979487103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3072,3072,0.024166399240493776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3072,2560,0.019421867529551187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3072,1536,0.013448533415794373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3072,1024,0.010478933652242024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3072,2048,0.01576959987481435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3072,768,0.009591466188430786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3072,512,0.0076799998680750535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3072,256,0.00628053347269694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3072,128,0.005256533126036326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3072,64,0.004300799965858459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3072,32,0.003549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,3072,65536,0.2815999984741211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2560,16384,0.07045119603474935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2560,6144,0.034542934099833174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2560,7168,0.03833173513412476
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2560,10240,0.048503466447194415
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2560,8192,0.042461868127187094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2560,12288,0.05717333157857259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2560,4096,0.026146133740743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2560,3584,0.022664533058802287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2560,5120,0.031573333342870075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2560,2560,0.017783466974894205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2560,3072,0.01976319948832194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2560,2048,0.014643200238545737
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2560,1536,0.012356266379356384
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2560,1024,0.009830400347709656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2560,768,0.008396800359090168
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2560,256,0.006007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2560,128,0.005051733553409576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2560,64,0.003993600110212962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2560,65536,0.23801172574361168
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2560,32,0.0035157332817713416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2560,512,0.0075434664885203045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2048,7168,0.035327998797098796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2048,6144,0.03027626673380534
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2048,8192,0.037034666538238524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2048,10240,0.04348586797714234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2048,5120,0.02542933424313863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2048,12288,0.04754773378372192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2048,16384,0.06191786527633667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2048,4096,0.020957867304484047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2048,2560,0.015701333681742348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2048,3584,0.019933867454528808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2048,1536,0.011059199770291645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2048,3072,0.017237333456675212
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2048,1024,0.008533333738644917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2048,2048,0.01259519954522451
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2048,65536,0.1979050636291504
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2048,128,0.00477866679430008
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2048,512,0.006451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2048,256,0.0054954667886098225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2048,32,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2048,64,0.003857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,2048,768,0.007645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1536,5120,0.019182932376861573
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1536,7168,0.02512213389078776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1536,10240,0.033928533395131424
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1536,6144,0.02211839954058329
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1536,8192,0.028637866179148357
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1536,12288,0.038126933574676516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1536,16384,0.049561599890391037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1536,3584,0.017783466974894205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1536,4096,0.01853440006573995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1536,3072,0.015940266847610473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1536,1536,0.010342400272687275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1536,2560,0.014028799533843995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1536,2048,0.011878400047620138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1536,1024,0.008021333316961924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1536,65536,0.15025493303934734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1536,768,0.007099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1536,128,0.004676266511281332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1536,256,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1536,64,0.00365226666132609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1536,32,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1536,512,0.006178133189678192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1024,16384,0.037478399276733396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1024,8192,0.021333332856496176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1024,10240,0.02590720057487488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1024,12288,0.029183999697367353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1024,6144,0.01812480092048645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1024,7168,0.019899733861287437
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1024,5120,0.01693013310432434
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1024,4096,0.013994666934013366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1024,3072,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1024,3584,0.013141333063443502
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1024,2560,0.011571199695269267
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1024,1536,0.00962559978167216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1024,2048,0.011502933502197266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1024,1024,0.007918933530648549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1024,65536,0.10635946591695149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1024,768,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1024,512,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1024,256,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1024,128,0.004232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1024,64,0.003549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,1024,32,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,768,16384,0.03160746693611145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,768,12288,0.023415466149648033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,768,8192,0.016622933745384216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,768,7168,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,768,10240,0.018329600493113198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,768,5120,0.012970667084058127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,768,6144,0.01378986636797587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,768,4096,0.012219732999801636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,768,65536,0.0859818696975708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,768,3584,0.012936533490816752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,768,2048,0.009454933802286784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,768,3072,0.011912533640861511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,768,2560,0.01129813293615977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,768,1024,0.007850666840871174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,768,1536,0.00962559978167216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,768,768,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,768,256,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,768,128,0.0037546666959921518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,768,64,0.0032085334261258446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,768,32,0.00314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,768,512,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,512,16384,0.022562134265899658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,512,10240,0.015735466281572977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,512,8192,0.014404267072677612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,512,6144,0.01242453356583913
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,512,7168,0.017100799083709716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,512,5120,0.011912533640861511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,512,12288,0.018602667252222697
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,512,4096,0.010444800059000652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,512,65536,0.0676522652308146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,512,3584,0.010001066327095031
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,512,3072,0.00955733358860016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,512,2560,0.009079466263453167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,512,2048,0.008396800359090168
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,512,1024,0.007782400151093801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,512,1536,0.007987200220425924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,512,768,0.006963199873765309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,512,512,0.006075733403364817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,512,256,0.004539733131726583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,512,128,0.003549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,512,32,0.0030037333567937215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,512,64,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,256,12288,0.012288000186284382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,256,10240,0.011434666315714518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,256,16384,0.014097066720326743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,256,7168,0.00993280013402303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,256,6144,0.009318400422732036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,256,5120,0.00897706647713979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,256,8192,0.010444800059000652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,256,65536,0.04194986820220947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,256,4096,0.008396800359090168
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,256,3584,0.007987200220425924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,256,3072,0.007748266557852428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,256,2560,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,256,2048,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,256,1536,0.006894933183987935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,256,1024,0.006587733328342438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,256,768,0.006894933183987935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,256,512,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,256,256,0.004334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,256,64,0.004095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,256,32,0.004198400179545084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,256,128,0.0033791999022165934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,128,16384,0.011059199770291645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,128,12288,0.00942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,128,10240,0.00897706647713979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,128,8192,0.008601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,128,65536,0.029388799269994097
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,128,7168,0.008123733103275299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,128,6144,0.007816533247629803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,128,5120,0.007577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,128,4096,0.007167999943097432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,128,3584,0.007782400151093801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,128,3072,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,128,2560,0.00747519979874293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,128,1536,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,128,1024,0.00628053347269694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,128,2048,0.006621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,128,768,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,128,512,0.006041599810123444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,128,128,0.0032426667710145317
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,128,64,0.0040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,128,32,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,128,256,0.004232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,64,16384,0.009079466263453167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,64,10240,0.008089600006739299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,64,7168,0.007782400151093801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,64,8192,0.007714133461316426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,64,12288,0.008430932958920796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,64,65536,0.015735466281572977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,64,6144,0.007577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,64,4096,0.007202133536338806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,64,5120,0.007714133461316426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,64,3584,0.007236266632874806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,64,1536,0.005973333120346069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,64,3072,0.006826666494210561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,64,2560,0.006690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,64,1024,0.005529599885145823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,64,2048,0.007099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,64,768,0.006792533397674561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,64,512,0.005905066430568695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,64,128,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,64,64,0.0040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,64,32,0.0040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,64,256,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,32,12288,0.008089600006739299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,32,16384,0.008738133311271667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,32,65536,0.015155200163523355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,32,8192,0.0076117331782976794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,32,7168,0.0075434664885203045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,32,6144,0.007406933108965556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,32,5120,0.007338666419188182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,32,10240,0.007816533247629803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,32,4096,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,32,3584,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,32,3072,0.006758399804433187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,32,2560,0.006621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,32,1536,0.005529599885145823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,32,2048,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,32,1024,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,32,512,0.005836800237496694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,32,768,0.006690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,32,256,0.005017599960168203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,32,128,0.004369066655635833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,32,64,0.004369066655635833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,128,32,32,0.0040277334551016486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,65536,7168,0.6533461252848307
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,65536,8192,0.729804801940918
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,65536,4096,0.3725994745890299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,65536,5120,0.4731904029846191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,65536,3584,0.36652374267578125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,65536,12288,1.0344789505004883
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,65536,6144,0.5616298675537109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,65536,3072,0.31976105372111
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,65536,1024,0.12270932992299396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,65536,512,0.06857386430104574
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,65536,1536,0.1755136013031006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,65536,128,0.030754133065541582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,65536,256,0.04355413516362508
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,65536,64,0.023825067281723022
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,65536,768,0.09663146336873372
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,65536,32,0.018227199713389076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,65536,2560,0.24746665954589844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,65536,2048,0.22432427406311034
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,65536,10240,0.8808789571126303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,16384,10240,0.236407470703125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,16384,16384,0.3554645220438639
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,16384,5120,0.1425066630045573
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,16384,7168,0.17370452880859374
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,16384,8192,0.21357226371765137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,16384,6144,0.16612693468729656
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,16384,12288,0.27436374028523763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,16384,4096,0.11922773520151775
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,16384,3584,0.10359466870625814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,16384,1024,0.03945813179016113
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,16384,1536,0.056934400399525964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,16384,2048,0.06649173498153686
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,16384,3072,0.09618773460388183
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,16384,2560,0.09168213208516439
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,16384,512,0.019899733861287437
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,16384,768,0.03092479904492696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,16384,256,0.013209600249926248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,16384,128,0.010205866893132527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,16384,64,0.008260266482830047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,16384,32,0.007338666419188182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,65536,16384,1.352635701497396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,12288,16384,0.27607040405273436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,12288,7168,0.1276245355606079
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,12288,8192,0.13957120577494303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,12288,10240,0.17466026941935223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,12288,12288,0.20507307052612306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,12288,5120,0.10547200043996174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,12288,6144,0.12291413148244221
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,12288,3584,0.07307946681976318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,12288,4096,0.07970133622487387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,12288,3072,0.060279464721679686
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,12288,2560,0.05410133202870687
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,12288,2048,0.04386133352915446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,16384,65536,1.3398357391357423
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,12288,1024,0.024541866779327393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,12288,768,0.019285333156585694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,12288,512,0.014677332838376364
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,12288,256,0.010786133011182149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,12288,1536,0.03345066706339518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,12288,64,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,12288,128,0.008123733103275299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,12288,32,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,10240,7168,0.11421013673146566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,10240,8192,0.12025173505147298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,10240,6144,0.09908906618754068
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,10240,10240,0.15131306648254395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,10240,12288,0.17974613507588705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,10240,4096,0.0703488032023112
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,10240,5120,0.08277333577473958
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,12288,65536,1.0252970377604167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,10240,16384,0.2285226662953695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,10240,2048,0.03942399819691976
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,10240,1536,0.03078826665878296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,10240,1024,0.021913599967956544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,10240,2560,0.0474453330039978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,10240,3072,0.05382826725641886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,10240,3584,0.060108800729115806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,10240,512,0.013619200388590495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,10240,768,0.017476266622543334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,10240,128,0.00747519979874293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,10240,256,0.00976213316122691
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,10240,64,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,10240,32,0.005188266436258951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,8192,10240,0.12516693274180096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,8192,16384,0.18807466824849445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,8192,8192,0.11192320187886555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,8192,12288,0.14349652926127116
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,8192,7168,0.09646080334981283
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,8192,6144,0.08823466300964355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,8192,4096,0.06184959808985392
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,8192,5120,0.079257599512736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,8192,3584,0.0582314650217692
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,8192,1536,0.027374933163324993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,8192,1024,0.019694934288660683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,8192,768,0.016076800227165223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,8192,2560,0.04358826478322347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,8192,3072,0.05174613396326701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,8192,2048,0.03413333495457967
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,8192,512,0.01262933313846588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,8192,256,0.009011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,8192,128,0.007133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,8192,64,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,8192,32,0.00477866679430008
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,7168,10240,0.11158186594645184
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,7168,12288,0.12653226852416993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,7168,16384,0.16913065910339356
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,7168,8192,0.13322239716847736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,7168,7168,0.0964949369430542
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,7168,4096,0.08331946531931558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,7168,6144,0.08458240032196045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,7168,5120,0.09775786399841309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,10240,65536,0.855722681681315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,7168,2048,0.03857066631317139
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,7168,2560,0.04792319933573405
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,7168,1536,0.03126613299051921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,7168,512,0.014062933127085366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,7168,768,0.017203199863433837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,7168,256,0.008840533097585042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,7168,3072,0.06830080350240073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,7168,3584,0.07700479825337728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,7168,1024,0.022732800245285033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,7168,128,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,7168,64,0.005256533126036326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,7168,32,0.0045056000351905824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,8192,65536,0.693998908996582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,6144,8192,0.07895039717356364
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,6144,10240,0.0982698678970337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,6144,7168,0.07570772965749105
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,6144,4096,0.0495957334836324
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,6144,5120,0.05683199961980184
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,6144,3584,0.04693333307902019
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,6144,12288,0.11799893379211426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,6144,16384,0.15977813402811686
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,6144,6144,0.07017813523610433
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,6144,3072,0.04123306671778361
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,6144,2560,0.028637866179148357
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,6144,2048,0.02215253313382467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,6144,1536,0.0184661328792572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,6144,1024,0.013960533340771995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,6144,512,0.009215999643007915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,6144,256,0.006690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,6144,768,0.011434666315714518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,6144,128,0.00580266664425532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,7168,65536,0.6057984034220378
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,6144,64,0.005120000243186951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,6144,32,0.004266666869322458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,5120,10240,0.08410453001658122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,5120,16384,0.1211733341217041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,5120,12288,0.09553919633229574
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,5120,6144,0.05935786565144857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,5120,7168,0.06679893334706624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,5120,8192,0.07608319918314616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,5120,5120,0.05485226710637411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,5120,4096,0.045431466897328694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,5120,3072,0.03068586587905884
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,5120,3584,0.034747731685638425
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,6144,65536,0.5228202819824219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,5120,2560,0.026077866554260254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,5120,2048,0.019899733861287437
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,5120,1536,0.017032533884048462
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,5120,1024,0.012834133704503379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,5120,768,0.010683733224868774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,5120,512,0.008499200145403545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,5120,256,0.006451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,5120,128,0.00576853354771932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,5120,64,0.004710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,5120,32,0.003857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,5120,65536,0.43291308085123703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,4096,6144,0.046011734008789065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,4096,5120,0.042837333679199216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,4096,8192,0.05870933135350546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,4096,7168,0.05249706506729126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,4096,10240,0.07062186400095621
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,4096,12288,0.08192000389099122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,4096,16384,0.10359466870625814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,4096,4096,0.03433813254038493
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,4096,2560,0.0233130673567454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,4096,3072,0.027272532383600872
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,4096,2048,0.01904639999071757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,4096,1536,0.015667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,4096,3584,0.03170986572901408
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,4096,1024,0.011946666240692138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,4096,768,0.010069333513577779
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,4096,128,0.0053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,4096,256,0.006041599810123444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,4096,512,0.007918933530648549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,4096,64,0.004300799965858459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,4096,32,0.003618133316437403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,4096,65536,0.35812692642211913
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3584,6144,0.04300800164540609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3584,16384,0.09632426897684733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3584,7168,0.04771840174992879
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3584,5120,0.03932160139083862
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3584,8192,0.06301013231277466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3584,4096,0.031300266583760575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3584,12288,0.0723626693089803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3584,10240,0.06372693379720053
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3584,3584,0.029661866029103597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3584,2560,0.02211839954058329
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3584,1536,0.015121066570281982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3584,3072,0.025941334168116253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3584,2048,0.018193066120147705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3584,1024,0.011468799908955891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3584,768,0.009523199995358785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3584,512,0.007748266557852428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3584,256,0.006041599810123444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3584,128,0.005051733553409576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3584,64,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3584,32,0.0037546666959921518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3072,8192,0.04877653519312541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3072,12288,0.06608213186264038
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3072,10240,0.05628586610158285
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3584,65536,0.31494827270507814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3072,16384,0.07983787059783935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3072,7168,0.045465600490570066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3072,5120,0.03280213276545207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3072,6144,0.03778560161590576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3072,3584,0.026146133740743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3072,4096,0.028910932938257854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3072,3072,0.023654399315516154
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3072,2560,0.018090667327245076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3072,1536,0.01269760032494863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3072,768,0.008499200145403545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3072,2048,0.014779733618100485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3072,1024,0.010240000486373902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3072,512,0.007099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3072,64,0.0039594667653242745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3072,256,0.005973333120346069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3072,32,0.0034815999368826545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3072,128,0.004915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,3072,65536,0.273578675587972
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2560,12288,0.05935786565144857
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2560,10240,0.04703573385874431
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2560,7168,0.03679573138554891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2560,8192,0.04133546749750773
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2560,16384,0.06935893694559733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2560,6144,0.03324586749076843
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2560,4096,0.025702399015426636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2560,5120,0.030583467086156207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2560,3584,0.023449599742889404
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2560,2560,0.016588800152142844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2560,2048,0.01378986636797587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2560,1536,0.01160533328851064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2560,3072,0.01853440006573995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2560,1024,0.009318400422732036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2560,512,0.006758399804433187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2560,768,0.007850666840871174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2560,256,0.005666133264700572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2560,128,0.004744533201058706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2560,64,0.0037546666959921518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2560,65536,0.23002452850341798
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2560,32,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2048,16384,0.05840213298797607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2048,12288,0.04836693207422892
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2048,10240,0.04092586835225423
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2048,7168,0.031436800956726074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2048,6144,0.029491200049718218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2048,8192,0.03399680058161418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2048,5120,0.024132267634073893
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2048,4096,0.021572266022364298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2048,3584,0.020172800620396933
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2048,3072,0.017407999436060587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2048,2560,0.014165332913398743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2048,1536,0.010444800059000652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2048,2048,0.012288000186284382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2048,768,0.007406933108965556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2048,1024,0.008294400076071422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2048,65536,0.18933760325113932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2048,512,0.00628053347269694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2048,256,0.0052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2048,128,0.00481279989083608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2048,32,0.0033791999022165934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,2048,64,0.0036864000062147772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1536,8192,0.027306665976842243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1536,10240,0.03222186764081319
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1536,7168,0.024541866779327393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1536,12288,0.03857066631317139
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1536,6144,0.021367466449737547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1536,5120,0.019592533508936562
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1536,16384,0.0451583981513977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1536,4096,0.01634986698627472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1536,3584,0.01655466655890147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1536,3072,0.014779733618100485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1536,2048,0.011673600474993388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1536,1536,0.009796266754468281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1536,2560,0.013755733768145243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1536,1024,0.00795306662718455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1536,65536,0.143940273920695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1536,768,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1536,512,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1536,256,0.005188266436258951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1536,128,0.0045056000351905824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1536,64,0.003549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1536,32,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1024,16384,0.03604480028152466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1024,10240,0.024268800020217897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1024,12288,0.027989333868026732
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1024,8192,0.019421867529551187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1024,7168,0.01802240014076233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1024,6144,0.016247466206550598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1024,5120,0.015530666708946228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1024,4096,0.01795413295427958
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1024,3584,0.012868266304334006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1024,3072,0.016759467124938966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1024,2560,0.011366400122642516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1024,2048,0.011434666315714518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1024,1536,0.009659733374913533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1024,65536,0.10550613403320312
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1024,1024,0.007884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1024,768,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1024,256,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1024,512,0.006109866499900818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1024,64,0.00365226666132609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1024,128,0.003822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,1024,32,0.0032768001159032187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,768,16384,0.02877440055211385
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,768,12288,0.022425599892934165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,768,8192,0.016076800227165223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,768,10240,0.017783466974894205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,768,7168,0.014711466431617738
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,768,6144,0.01372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,768,5120,0.01269760032494863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,768,3584,0.012219732999801636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,768,3072,0.011434666315714518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,768,2560,0.010513066252072652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,768,2048,0.009011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,768,4096,0.011434666315714518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,768,1536,0.009591466188430786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,768,65536,0.08226133187611898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,768,1024,0.007850666840871174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,768,768,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,768,512,0.006109866499900818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,768,256,0.004642133414745331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,768,128,0.003618133316437403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,768,32,0.00314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,768,64,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,512,16384,0.020206934213638304
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,512,12288,0.016383999586105348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,512,8192,0.014916266997655234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,512,7168,0.014165332913398743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,512,10240,0.015871999661127727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,512,6144,0.012902399897575379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,512,5120,0.012083199620246888
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,512,4096,0.01129813293615977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,512,65536,0.0695637305577596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,512,3072,0.010171733299891154
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,512,2560,0.010342400272687275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,512,2048,0.009318400422732036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,512,3584,0.010786133011182149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,512,1536,0.008635733524958293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,512,1024,0.007850666840871174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,512,768,0.013004799683888754
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,512,512,0.005973333120346069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,512,32,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,512,256,0.004437333345413208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,512,128,0.0034133332471052804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,512,64,0.0032768001159032187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,256,12288,0.012014933427174886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,256,16384,0.013858133554458618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,256,65536,0.03932160139083862
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,256,10240,0.011332266529401143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,256,5120,0.008874666690826417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,256,6144,0.009250133236249288
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,256,8192,0.010240000486373902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,256,7168,0.009693866968154908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,256,4096,0.008021333316961924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,256,3584,0.007884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,256,3072,0.0075434664885203045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,256,2048,0.006894933183987935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,256,1536,0.006724266707897187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,256,768,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,256,2560,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,256,1024,0.006382933259010315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,256,512,0.005905066430568695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,256,128,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,256,64,0.004095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,256,32,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,256,256,0.004334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,128,12288,0.009284266829490661
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,128,16384,0.010513066252072652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,128,65536,0.028194133440653486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,128,10240,0.008772266904513042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,128,7168,0.008021333316961924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,128,6144,0.0076799998680750535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,128,8192,0.008430932958920796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,128,4096,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,128,5120,0.007406933108965556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,128,3584,0.0076117331782976794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,128,3072,0.006929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,128,2560,0.007406933108965556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,128,2048,0.006621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,128,1536,0.0065536002318064375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,128,1024,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,128,768,0.006826666494210561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,128,512,0.006007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,128,256,0.004266666869322458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,128,128,0.004437333345413208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,128,64,0.004095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,128,32,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,64,16384,0.00890880028406779
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,64,12288,0.008294400076071422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,64,65536,0.015359999736150107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,64,7168,0.0076799998680750535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,64,10240,0.008021333316961924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,64,8192,0.007748266557852428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,64,6144,0.00747519979874293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,64,5120,0.0076799998680750535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,64,4096,0.007133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,64,3584,0.007167999943097432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,64,2560,0.006929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,64,3072,0.006758399804433187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,64,1024,0.0052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,64,768,0.006792533397674561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,64,2048,0.006894933183987935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,64,512,0.005870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,64,256,0.005051733553409576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,64,1536,0.005836800237496694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,64,128,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,64,64,0.003993600110212962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,64,32,0.004130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,32,16384,0.008533333738644917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,32,12288,0.008021333316961924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,32,65536,0.014916266997655234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,32,7168,0.007577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,32,6144,0.0073728000124295544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,32,10240,0.007748266557852428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,32,5120,0.00730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,32,8192,0.0076117331782976794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,32,3584,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,32,4096,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,32,3072,0.006690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,32,2560,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,32,2048,0.006621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,32,1536,0.0056320001681645715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,32,1024,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,32,768,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,32,512,0.00576853354771932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,32,128,0.004334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,32,64,0.003993600110212962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,32,256,0.004983466863632202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,96,32,32,0.0040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,65536,5120,0.4697429339090983
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,65536,3584,0.33423360188802087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,65536,4096,0.36980053583780925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,65536,10240,0.8768853505452474
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,65536,6144,0.5595136006673177
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,65536,2048,0.21862400372823082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,65536,3072,0.29081598917643225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,65536,7168,0.6482602437337239
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,65536,1024,0.12062719662984211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,65536,768,0.09550506273905436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,65536,16384,1.3438634236653646
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,65536,2560,0.2461695988972982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,65536,256,0.04355413516362508
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,65536,512,0.06761813163757324
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,65536,1536,0.1731584072113037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,65536,128,0.03037866751352946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,65536,8192,0.7338666915893555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,65536,64,0.024234666426976522
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,65536,32,0.0184661328792572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,16384,7168,0.1750698725382487
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,16384,10240,0.23497385978698732
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,16384,4096,0.11677013238271076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,16384,6144,0.15336106618245443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,16384,5120,0.13209599653879803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,16384,8192,0.21599574089050294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,65536,12288,1.0286421457926431
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,16384,12288,0.2723157246907552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,16384,2560,0.0923306703567505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,16384,3584,0.10284372965494792
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,16384,16384,0.3517439842224121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,16384,2048,0.06693546772003174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,16384,1024,0.03857066631317139
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,16384,3072,0.09564159711201986
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,16384,1536,0.05461333195368449
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,16384,768,0.029388799269994097
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,16384,256,0.01372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,16384,512,0.021367466449737547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,16384,64,0.008123733103275299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,16384,128,0.010035199920336406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,16384,32,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,12288,10240,0.17257812817891438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,12288,16384,0.26654720306396484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,12288,6144,0.10881706873575847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,12288,8192,0.13806932767232258
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,12288,7168,0.12653226852416993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,12288,5120,0.09936213493347168
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,12288,12288,0.20753067334493003
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,12288,4096,0.07256746292114258
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,12288,3072,0.06331733465194703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,12288,2560,0.04949333270390828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,12288,3584,0.06662826538085938
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,12288,2048,0.04389546712239583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,12288,1024,0.024268800020217897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,12288,1536,0.03426986535390218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,12288,128,0.0076117331782976794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,12288,64,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,12288,256,0.0105813334385554
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,12288,768,0.019217065970102944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,12288,512,0.013380266229311624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,12288,32,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,16384,65536,1.376324208577474
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,10240,16384,0.2295466740926107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,10240,6144,0.10373120307922364
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,10240,8192,0.11939840316772461
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,10240,7168,0.11735040346781414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,10240,10240,0.15544320742289225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,10240,12288,0.17807359695434571
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,10240,5120,0.08751786549886068
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,10240,4096,0.06492160161336263
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,10240,2560,0.047172268231709794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,10240,3072,0.05573973258336386
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,10240,3584,0.059767464796702065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,10240,1536,0.030481066306432086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,10240,1024,0.019797333081563315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,10240,2048,0.03839999834696452
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,10240,128,0.007167999943097432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,10240,64,0.005973333120346069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,10240,768,0.017373865842819212
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,10240,512,0.013448533415794373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,10240,256,0.009352533022562663
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,10240,32,0.005154133339722951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,8192,16384,0.18623147010803223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,8192,8192,0.11185493469238281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,8192,7168,0.09427626927693686
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,8192,10240,0.1253717343012492
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,8192,12288,0.14298453330993652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,10240,65536,0.8533674875895182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,12288,65536,1.0209962844848632
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,8192,6144,0.0932522694269816
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,8192,3072,0.050722134113311765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,8192,3584,0.056866133213043214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,8192,2560,0.04266666571299235
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,8192,4096,0.05758293469746908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,8192,5120,0.07287466526031494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,8192,2048,0.03461120128631592
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,8192,1536,0.02696533401807149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,8192,1024,0.01952426632245382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,8192,768,0.016008533040682473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,8192,256,0.009045333663622538
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,8192,128,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,8192,512,0.01269760032494863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,8192,32,0.004915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,8192,64,0.006007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,7168,8192,0.10328746636708577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,7168,7168,0.09321813583374024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,8192,65536,0.6878208160400391
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,7168,10240,0.10939733187357585
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,7168,12288,0.12564480304718018
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,7168,5120,0.0718506654103597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,7168,6144,0.08260266780853272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,7168,4096,0.05894826650619507
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,7168,16384,0.16414720217386883
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,7168,3584,0.07069013118743897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,7168,2048,0.03758080005645752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,7168,3072,0.060347731908162436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,7168,1536,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,7168,2560,0.05536426703135172
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,7168,512,0.013926399747530618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,7168,768,0.01904639999071757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,7168,1024,0.022664533058802287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,7168,256,0.008840533097585042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,7168,128,0.006929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,7168,64,0.005154133339722951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,7168,32,0.004437333345413208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,6144,10240,0.09881599744160971
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,6144,12288,0.11332266330718994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,6144,6144,0.06471680005391439
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,6144,16384,0.144759464263916
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,6144,7168,0.07226026852925618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,6144,8192,0.0771071990331014
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,6144,5120,0.058641068140665685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,6144,4096,0.05208746592203776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,6144,1536,0.017271467049916587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,6144,1024,0.013926399747530618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,6144,2048,0.023381332556406655
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,6144,2560,0.028842665751775104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,6144,3584,0.04382719993591309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,6144,3072,0.04140373468399048
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,7168,65536,0.6014293034871419
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,6144,768,0.010922666390736897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,6144,512,0.009079466263453167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,6144,64,0.005120000243186951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,6144,128,0.0054954667886098225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,6144,256,0.0065536002318064375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,6144,32,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,6144,65536,0.522649606068929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,5120,12288,0.09482239882151286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,5120,16384,0.12052480379740398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,5120,6144,0.05867520173390707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,5120,7168,0.06516053279240927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,5120,8192,0.07000746726989746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,5120,10240,0.08192000389099122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,5120,5120,0.053896534442901614
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,5120,3584,0.032767999172210696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,5120,4096,0.0451583981513977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,5120,3072,0.03041279911994934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,5120,2560,0.02604373296101888
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,5120,2048,0.021230934063593547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,5120,1024,0.011844266454378765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,5120,1536,0.01689599951108297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,5120,768,0.010137599706649781
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,5120,512,0.008499200145403545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,5120,32,0.003822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,5120,256,0.006451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,5120,128,0.005666133264700572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,5120,64,0.004744533201058706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,4096,10240,0.0688810666402181
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,4096,12288,0.07707306543986002
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,4096,16384,0.10263893604278565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,4096,8192,0.0582314650217692
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,4096,5120,0.04194986820220947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,4096,6144,0.046216531594594316
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,4096,7168,0.051370668411254886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,4096,4096,0.034918399651845296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,4096,3584,0.03109546701113383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,4096,2560,0.021811199188232423
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,5120,65536,0.43001171747843425
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,4096,3072,0.027170133590698243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,4096,2048,0.01925119956334432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,4096,1536,0.015564800302187601
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,4096,1024,0.011502933502197266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,4096,768,0.00976213316122691
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,4096,512,0.0076799998680750535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,4096,256,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,4096,128,0.005188266436258951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,4096,64,0.004198400179545084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,4096,32,0.00365226666132609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3584,16384,0.09530026912689209
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3584,12288,0.07212373415629068
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,4096,65536,0.35543041229248046
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3584,6144,0.04126720031102498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3584,8192,0.05311146577199301
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3584,10240,0.06283946832021078
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3584,7168,0.04686506589253743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3584,4096,0.031880533695220946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3584,5120,0.0370688001314799
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3584,3584,0.02945706645647685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3584,2048,0.017578667402267455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3584,2560,0.021947733561197915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3584,3072,0.024268800020217897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3584,1536,0.015086932977040609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3584,1024,0.011161599556605022
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3584,512,0.007645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3584,768,0.009454933802286784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3584,256,0.006007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3584,128,0.005051733553409576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3584,64,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3584,32,0.003618133316437403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3584,65536,0.3088383992513021
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3072,12288,0.06526293357213339
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3072,7168,0.04092586835225423
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3072,8192,0.04430506626764934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3072,5120,0.03362133502960205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3072,6144,0.037034666538238524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3072,10240,0.05536426703135172
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3072,16384,0.07942826747894287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3072,4096,0.027306665976842243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3072,3584,0.025804799795150758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3072,3072,0.022971733411153158
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3072,1536,0.012492799758911132
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3072,2048,0.014643200238545737
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3072,1024,0.009693866968154908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3072,2560,0.017407999436060587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3072,512,0.006963199873765309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3072,256,0.005905066430568695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3072,128,0.004881066580613455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3072,768,0.008294400076071422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3072,64,0.003925333420435587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3072,32,0.0035157332817713416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,3072,65536,0.27351039250691733
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2560,8192,0.040516265233357746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2560,5120,0.029388799269994097
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2560,6144,0.03218773404757182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2560,7168,0.03566933472951253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2560,10240,0.045772798856099445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2560,12288,0.05358933210372925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2560,16384,0.06853973070780436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2560,3584,0.021128533283869426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2560,4096,0.023108265797297158
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2560,3072,0.01914880077044169
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2560,2048,0.01372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2560,2560,0.016145066420237223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2560,1536,0.01153706709543864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2560,1024,0.009181867043177288
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2560,65536,0.22828373908996583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2560,768,0.007816533247629803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2560,256,0.0054613331953684485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2560,128,0.004710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2560,512,0.006451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2560,64,0.0036864000062147772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2560,32,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2048,7168,0.03065173427263896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2048,12288,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2048,16384,0.06140586535135904
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2048,5120,0.023654399315516154
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2048,6144,0.027409066756566364
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2048,8192,0.03321173389752706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2048,10240,0.0400383989016215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2048,3584,0.018568533658981323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2048,3072,0.01628159979979197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2048,4096,0.01966080069541931
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2048,2560,0.013994666934013366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2048,2048,0.011946666240692138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2048,1536,0.010240000486373902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2048,1024,0.008260266482830047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2048,512,0.00628053347269694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2048,768,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2048,256,0.0053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2048,128,0.0045738667249679565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2048,32,0.0033791999022165934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2048,65536,0.18763093948364257
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,2048,64,0.0037205333511034647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1536,12288,0.0362496018409729
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1536,16384,0.044544001420338944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1536,10240,0.031982932488123575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1536,7168,0.023961599667867026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1536,8192,0.026692267258961993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1536,5120,0.019421867529551187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1536,6144,0.022801067431767783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1536,4096,0.016145066420237223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1536,3584,0.015359999736150107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1536,3072,0.014643200238545737
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1536,2560,0.013516799608866373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1536,2048,0.01160533328851064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1536,1024,0.007782400151093801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1536,1536,0.009796266754468281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1536,512,0.006178133189678192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1536,65536,0.14284799893697103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1536,768,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1536,256,0.005256533126036326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1536,128,0.004334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1536,64,0.0034815999368826545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1536,32,0.003583999971548716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1024,16384,0.033587201436360674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1024,10240,0.02368853290875753
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1024,12288,0.028842665751775104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1024,8192,0.01911466717720032
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1024,4096,0.013243732849756875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1024,7168,0.017885865767796834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1024,6144,0.016076800227165223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1024,5120,0.015291733543078103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1024,3584,0.01269760032494863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1024,3072,0.011844266454378765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1024,65536,0.10113706588745117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1024,2048,0.011468799908955891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1024,2560,0.010854400197664897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1024,1536,0.009591466188430786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1024,1024,0.007884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1024,768,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1024,512,0.006178133189678192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1024,256,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1024,64,0.0035157332817713416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1024,128,0.0036864000062147772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,1024,32,0.0031744000812371576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,768,10240,0.017578667402267455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,768,16384,0.02792106668154399
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,768,12288,0.021333332856496176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,768,7168,0.01460906664530436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,768,8192,0.015735466281572977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,768,6144,0.013482667009035745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,768,5120,0.01259519954522451
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,768,4096,0.012492799758911132
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,768,3584,0.012049067020416259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,768,3072,0.011229866743087768
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,768,65536,0.08034986654917399
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,768,2560,0.010444800059000652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,768,2048,0.008840533097585042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,768,1536,0.009591466188430786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,768,768,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,768,1024,0.007850666840871174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,768,512,0.006041599810123444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,768,256,0.00477866679430008
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,768,128,0.003583999971548716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,768,64,0.0033791999022165934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,768,32,0.0030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,512,16384,0.020241065820058187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,512,6144,0.012731732924779257
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,512,12288,0.016315733393033348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,512,10240,0.015667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,512,7168,0.014062933127085366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,512,8192,0.014847999811172486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,512,5120,0.011844266454378765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,512,65536,0.062327468395233156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,512,4096,0.011502933502197266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,512,2048,0.009181867043177288
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,512,2560,0.010137599706649781
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,512,3072,0.010035199920336406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,512,1536,0.008430932958920796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,512,1024,0.0076799998680750535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,512,3584,0.010444800059000652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,512,768,0.006929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,512,512,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,512,256,0.004437333345413208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,512,128,0.0033791999022165934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,512,32,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,512,64,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,256,12288,0.012014933427174886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,256,10240,0.011161599556605022
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,256,65536,0.040243200461069745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,256,16384,0.013823999961217245
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,256,7168,0.009591466188430786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,256,8192,0.010205866893132527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,256,6144,0.009215999643007915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,256,5120,0.008874666690826417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,256,4096,0.00795306662718455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,256,3072,0.00747519979874293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,256,3584,0.0076799998680750535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,256,2048,0.006826666494210561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,256,1536,0.006690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,256,1024,0.006348800162474315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,256,2560,0.007236266632874806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,256,768,0.006894933183987935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,256,512,0.006109866499900818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,256,256,0.004369066655635833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,256,128,0.0033791999022165934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,256,64,0.004095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,256,32,0.004198400179545084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,128,65536,0.025224532683690386
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,128,12288,0.009215999643007915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,128,8192,0.008499200145403545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,128,10240,0.008738133311271667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,128,7168,0.007850666840871174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,128,5120,0.007441066702206929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,128,6144,0.007645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,128,16384,0.01027413308620453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,128,4096,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,128,3072,0.006894933183987935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,128,2560,0.00747519979874293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,128,1536,0.006587733328342438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,128,2048,0.0065536002318064375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,128,3584,0.0076799998680750535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,128,1024,0.006212266782919565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,128,768,0.006826666494210561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,128,512,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,128,256,0.004198400179545084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,128,128,0.0044714664419492085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,128,64,0.0040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,128,32,0.004130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,64,16384,0.008874666690826417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,64,12288,0.008260266482830047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,64,65536,0.015359999736150107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,64,8192,0.007645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,64,10240,0.008021333316961924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,64,6144,0.007577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,64,7168,0.0076117331782976794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,64,4096,0.007133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,64,5120,0.0075434664885203045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,64,2560,0.006894933183987935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,64,2048,0.006826666494210561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,64,3584,0.007133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,64,1536,0.005870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,64,1024,0.0052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,64,3072,0.006792533397674561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,64,768,0.006792533397674561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,64,512,0.005905066430568695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,64,256,0.004983466863632202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,64,32,0.004130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,64,128,0.004369066655635833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,32,12288,0.007918933530648549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,32,10240,0.007714133461316426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,32,16384,0.00846506655216217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,32,65536,0.014847999811172486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,64,64,0.0040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,32,8192,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,32,7168,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,32,6144,0.00730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,32,5120,0.007406933108965556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,32,4096,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,32,3584,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,32,3072,0.006621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,32,2560,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,32,2048,0.006690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,32,1536,0.005666133264700572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,32,1024,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,32,768,0.006621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,32,512,0.006041599810123444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,32,256,0.004983466863632202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,32,128,0.004334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,32,64,0.0040277334551016486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,80,32,32,0.0040277334551016486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,65536,5120,0.42400426864624025
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,65536,7168,0.6467925389607747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,65536,8192,0.728610102335612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,65536,4096,0.3348479906717936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,65536,3584,0.30122667948404946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,65536,3072,0.2850816090901693
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,65536,2560,0.24101546605428062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,65536,768,0.07990612983703613
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,65536,1536,0.1470805327097575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,65536,1024,0.1008639971415202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,65536,256,0.03420159816741943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,65536,128,0.01935360034306844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,65536,6144,0.5044565200805664
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,65536,512,0.05505706469217936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,65536,32,0.01437013347943624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,65536,64,0.01532586713631948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,65536,2048,0.18804052670796711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,16384,10240,0.21998933156331382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,65536,10240,0.8691370646158854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,16384,12288,0.25883307456970217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,16384,16384,0.35382614135742185
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,16384,4096,0.0979967991511027
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,16384,5120,0.1262933333714803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,16384,7168,0.1574570655822754
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,16384,8192,0.18097492853800456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,16384,3584,0.0828757365544637
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,16384,3072,0.07976960341135661
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,65536,12288,1.0208255767822265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,16384,6144,0.15103999773661297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,16384,2560,0.06655999819437662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,16384,2048,0.05447680155436198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,16384,1536,0.04201813141504924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,16384,1024,0.02955946723620097
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,16384,256,0.010956799983978272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,16384,64,0.005973333120346069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,16384,768,0.023005867004394533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,16384,32,0.0054613331953684485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,16384,512,0.01518933375676473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,16384,128,0.007167999943097432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,65536,16384,1.3602133433024088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,12288,10240,0.17384106318155926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,12288,16384,0.26675198872884115
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,12288,8192,0.13851307233174642
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,12288,7168,0.12735146681467693
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,12288,5120,0.09055573145548503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,12288,4096,0.07246507008870443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,12288,12288,0.20759894053141276
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,12288,6144,0.10714453061421711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,12288,2560,0.04768426815668742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,12288,3584,0.06021120150883993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,12288,2048,0.03949226538340251
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,12288,3072,0.056524801254272464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,12288,1536,0.031300266583760575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,12288,1024,0.02044586737950643
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,12288,768,0.017100799083709716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,12288,512,0.01160533328851064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,12288,256,0.0073728000124295544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,12288,128,0.00576853354771932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,12288,64,0.005973333120346069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,12288,32,0.00477866679430008
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,10240,12288,0.1574570655822754
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,10240,8192,0.11263999938964844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,10240,7168,0.09625600179036459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,10240,6144,0.09089706738789877
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,10240,10240,0.1356800079345703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,10240,16384,0.21032959620157876
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,12288,65536,0.9878186543782552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,10240,3584,0.05502293507258097
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,10240,4096,0.05765120188395182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,10240,3072,0.04833279848098755
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,10240,2048,0.03317760030428569
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,10240,1536,0.026077866554260254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,10240,5120,0.07048532962799073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,10240,2560,0.040755200386047366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,10240,256,0.006587733328342438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,10240,1024,0.018602667252222697
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,10240,768,0.014711466431617738
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,10240,128,0.0055637334783871974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,10240,64,0.005154133339722951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,10240,512,0.010342400272687275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,16384,65536,1.4957567850748696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,10240,32,0.004369066655635833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,8192,6144,0.06802773475646973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,8192,8192,0.09168213208516439
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,10240,65536,0.8643242518107096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,8192,5120,0.05717333157857259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,8192,10240,0.11543893019358317
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,8192,7168,0.08663040002187093
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,8192,12288,0.13636266390482585
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,8192,3584,0.046830932299296066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,8192,4096,0.045875199635823566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,8192,16384,0.17960960070292156
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,8192,3072,0.0425983985265096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,8192,2560,0.033928533395131424
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,8192,1024,0.014711466431617738
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,8192,1536,0.021913599967956544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,8192,2048,0.026077866554260254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,8192,512,0.010035199920336406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,8192,256,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,8192,768,0.011776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,8192,64,0.004881066580613455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,8192,128,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,8192,32,0.003993600110212962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,7168,12288,0.12014933427174886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,7168,8192,0.08400213718414307
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,7168,10240,0.10332159996032715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,7168,5120,0.05761706829071045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,7168,7168,0.07137280305226644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,7168,4096,0.04491946697235107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,7168,6144,0.06184959808985392
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,7168,3584,0.039662933349609374
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,7168,16384,0.15940267244974773
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,7168,3072,0.03672746817270915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,7168,2048,0.024678399165471397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,7168,1024,0.013550933202107748
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,7168,1536,0.020002132654190062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,7168,512,0.009454933802286784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,7168,2560,0.031163734197616578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,7168,768,0.012049067020416259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,7168,256,0.006212266782919565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,7168,128,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,7168,32,0.003788800040880839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,7168,64,0.004676266511281332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,8192,65536,0.6849536259969076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,6144,12288,0.10386772950490315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,6144,8192,0.07679999669392903
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,6144,10240,0.08994133472442627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,6144,4096,0.03969706694285075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,6144,5120,0.04843519926071167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,6144,6144,0.061269334952036535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,6144,3584,0.0425983985265096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,6144,16384,0.1345194657643636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,6144,7168,0.07215786774953206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,6144,3072,0.031061333417892457
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,6144,2560,0.028194133440653486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,6144,1536,0.018295466899871826
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,6144,512,0.008294400076071422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,6144,1024,0.013653332988421122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,6144,768,0.011264000336329143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,6144,2048,0.02109439969062805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,6144,256,0.00628053347269694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,6144,128,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,6144,64,0.004232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,7168,65536,0.5827925364176433
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,6144,32,0.0036864000062147772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,5120,12288,0.08953173160552978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,5120,8192,0.05980159838994344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,5120,10240,0.07058773040771485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,5120,4096,0.03372373183568318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,5120,7168,0.05488640069961548
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,5120,5120,0.03997013171513875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,5120,16384,0.11598506768544514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,5120,6144,0.04341760079065959
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,5120,3584,0.029491200049718218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,5120,3072,0.02488320072491964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,5120,2048,0.0184661328792572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,6144,65536,0.4939093271891276
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,5120,1536,0.013926399747530618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,5120,2560,0.020889600118001304
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,5120,768,0.00976213316122691
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,5120,1024,0.01153706709543864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,5120,512,0.007850666840871174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,5120,256,0.006007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,5120,128,0.005017599960168203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,5120,64,0.004232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,5120,32,0.0037546666959921518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,4096,12288,0.07260159651438394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,4096,10240,0.06717440287272135
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,4096,16384,0.09560746351877848
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,4096,5120,0.0338261326154073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,4096,7168,0.05058559974034628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,4096,8192,0.05710506836573283
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,4096,6144,0.03696639935175578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,4096,4096,0.027511467536290485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,5120,65536,0.41649494171142576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,4096,3072,0.021196800470352172
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,4096,3584,0.02529279987017314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,4096,2560,0.019694934288660683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,4096,2048,0.016315733393033348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,4096,1536,0.013448533415794373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,4096,768,0.008157866696516674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,4096,1024,0.010410666465759277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,4096,256,0.0054954667886098225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,4096,512,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,4096,64,0.0040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,4096,128,0.00477866679430008
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,4096,32,0.008055466910203297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,4096,65536,0.33723732630411785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3584,10240,0.05877759853998819
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3584,16384,0.08789333502451578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3584,7168,0.044544001420338944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3584,5120,0.03126613299051921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3584,8192,0.04986879825592041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3584,6144,0.03553280035654704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3584,12288,0.07150932947794596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3584,4096,0.024780799945195518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3584,3584,0.02307413419087728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3584,2560,0.016827734311421712
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3584,3072,0.0212991992632548
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3584,2048,0.015291733543078103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3584,1536,0.012288000186284382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3584,768,0.007577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3584,1024,0.009523199995358785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3584,512,0.0065194666385650635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3584,256,0.0054613331953684485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3584,128,0.004676266511281332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3584,64,0.0040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3584,32,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3072,12288,0.059528533617655435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3072,16384,0.07365972995758056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3072,10240,0.05232640107472738
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3072,8192,0.04478293259938558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3584,65536,0.29805227915445964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3072,7168,0.039628799756368
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3072,6144,0.03563520113627116
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3072,5120,0.03310933311780294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3072,4096,0.02269866665204366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3072,3072,0.019729065895080566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3072,3584,0.020992000897725425
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3072,2048,0.01454080045223236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3072,1536,0.01129813293615977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3072,2560,0.017237333456675212
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3072,1024,0.008874666690826417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3072,768,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3072,512,0.006178133189678192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3072,65536,0.26241706212361654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3072,128,0.0044714664419492085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3072,256,0.0053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3072,64,0.003788800040880839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,3072,32,0.0032768001159032187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2560,10240,0.04392960071563721
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2560,7168,0.031470932563145954
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2560,6144,0.02815999984741211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2560,5120,0.023995733261108397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2560,8192,0.03519146839777629
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2560,12288,0.04942506551742554
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2560,16384,0.06304426590601603
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2560,4096,0.020002132654190062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2560,3072,0.016042666633923848
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2560,2560,0.014301866292953491
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2560,2048,0.012800000111262002
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2560,1536,0.011468799908955891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2560,3584,0.01884160041809082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2560,1024,0.008226133386294047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2560,768,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2560,128,0.004334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2560,64,0.0037205333511034647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2560,512,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2560,256,0.005188266436258951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2560,32,0.0034133332471052804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2560,65536,0.21889707247416176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2048,16384,0.05649066766103109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2048,5120,0.022050132354100548
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2048,10240,0.034918399651845296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2048,7168,0.027033599217732747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2048,6144,0.023756800095240276
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2048,8192,0.030037333567937214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2048,12288,0.04116479953130086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2048,4096,0.016793600718180337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2048,3584,0.015701333681742348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2048,3072,0.01546239952246348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2048,2560,0.013038933277130127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2048,2048,0.011707733074824016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2048,1536,0.010171733299891154
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2048,1024,0.007987200220425924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2048,768,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2048,65536,0.18036053975423177
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2048,512,0.006178133189678192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2048,256,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2048,128,0.009250133236249288
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2048,64,0.0038912000755469004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,2048,32,0.0034133332471052804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1536,16384,0.0400383989016215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1536,10240,0.028194133440653486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1536,12288,0.034815998872121175
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1536,8192,0.0237226665019989
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1536,6144,0.01843199928601583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1536,7168,0.020992000897725425
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1536,5120,0.016008533040682473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1536,4096,0.014097066720326743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1536,3584,0.013380266229311624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1536,3072,0.01242453356583913
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1536,2560,0.011912533640861511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1536,1536,0.00962559978167216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1536,65536,0.13499733606974285
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1536,1024,0.007850666840871174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1536,2048,0.01160533328851064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1536,768,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1536,512,0.006178133189678192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1536,256,0.005188266436258951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1536,128,0.003788800040880839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1536,64,0.003822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1536,32,0.0031744000812371576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1024,12288,0.024166399240493776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1024,16384,0.03242666721343994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1024,10240,0.020138667027155558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1024,7168,0.015633066495259605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1024,8192,0.017203199863433837
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1024,6144,0.014404267072677612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1024,5120,0.0130730668703715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1024,4096,0.011844266454378765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1024,3584,0.011264000336329143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1024,65536,0.09782613118489583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1024,2560,0.009352533022562663
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1024,2048,0.008396800359090168
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1024,1024,0.007816533247629803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1024,1536,0.009591466188430786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1024,3072,0.010376532872517902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1024,768,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1024,256,0.004710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1024,512,0.006041599810123444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1024,128,0.003549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1024,64,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,1024,32,0.0030720000465710956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,768,12288,0.023449599742889404
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,768,16384,0.023859200874964397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,768,10240,0.016076800227165223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,768,8192,0.014097066720326743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,768,7168,0.012902399897575379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,768,4096,0.009659733374913533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,768,6144,0.01181013286113739
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,768,5120,0.011127466956774395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,768,3584,0.009386666615804036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,768,65536,0.0740010658899943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,768,3072,0.008738133311271667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,768,2048,0.008021333316961924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,768,1024,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,768,768,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,768,1536,0.007577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,768,2560,0.008533333738644917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,768,256,0.004539733131726583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,768,512,0.005973333120346069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,768,128,0.0034815999368826545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,768,64,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,768,32,0.004198400179545084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,512,16384,0.017066667477289833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,512,10240,0.012970667084058127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,512,12288,0.014199466506640116
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,512,8192,0.011844266454378765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,512,7168,0.011059199770291645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,512,6144,0.009898666540781658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,512,5120,0.009079466263453167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,512,65536,0.06041599909464518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,512,4096,0.008294400076071422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,512,3584,0.008021333316961924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,512,2560,0.007577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,512,3072,0.007782400151093801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,512,2048,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,512,1536,0.006929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,512,1024,0.0065536002318064375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,512,768,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,512,512,0.006451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,512,256,0.004369066655635833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,512,32,0.004198400179545084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,512,128,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,256,12288,0.009864532947540283
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,256,16384,0.011571199695269267
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,256,10240,0.009079466263453167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,512,64,0.020002132654190062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,256,65536,0.03413333495457967
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,256,8192,0.008601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,256,7168,0.008089600006739299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,256,4096,0.007133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,256,5120,0.0075434664885203045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,256,3584,0.007714133461316426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,256,6144,0.007782400151093801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,256,2560,0.0075434664885203045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,256,3072,0.006963199873765309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,256,2048,0.006587733328342438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,256,1536,0.006587733328342438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,256,1024,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,256,768,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,256,512,0.006041599810123444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,256,256,0.004232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,256,128,0.0034133332471052804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,256,32,0.004130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,256,64,0.0040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,128,12288,0.008396800359090168
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,128,16384,0.009181867043177288
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,128,10240,0.008123733103275299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,128,7168,0.007850666840871174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,128,65536,0.01949013272921244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,128,8192,0.007884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,128,6144,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,128,4096,0.007202133536338806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,128,3584,0.007202133536338806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,128,3072,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,128,5120,0.0076799998680750535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,128,2560,0.006758399804433187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,128,2048,0.008874666690826417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,128,1536,0.006075733403364817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,128,1024,0.0054613331953684485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,128,768,0.006826666494210561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,128,512,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,128,256,0.004198400179545084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,128,64,0.004095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,128,32,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,128,128,0.009727999567985535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,64,16384,0.010171733299891154
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,64,10240,0.008772266904513042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,64,12288,0.008874666690826417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,64,8192,0.008567466338475545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,64,7168,0.008123733103275299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,64,65536,0.013653332988421122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,64,6144,0.007714133461316426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,64,4096,0.007816533247629803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,64,3072,0.0065536002318064375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,64,3584,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,64,2560,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,64,2048,0.006724266707897187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,64,1536,0.0055637334783871974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,64,1024,0.005256533126036326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,64,5120,0.007441066702206929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,64,768,0.006758399804433187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,64,512,0.005836800237496694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,64,256,0.005017599960168203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,64,64,0.003993600110212962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,64,128,0.004369066655635833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,64,32,0.004095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,32,12288,0.008738133311271667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,32,16384,0.009045333663622538
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,32,65536,0.012492799758911132
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,32,10240,0.008533333738644917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,32,8192,0.008260266482830047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,32,7168,0.008021333316961924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,32,6144,0.007577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,32,5120,0.00747519979874293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,32,4096,0.0076117331782976794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,32,3072,0.0065194666385650635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,32,2560,0.006621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,32,2048,0.0065194666385650635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,32,1536,0.005529599885145823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,32,3584,0.007441066702206929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,32,1024,0.005154133339722951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,32,512,0.00576853354771932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,32,256,0.004949333270390829
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,32,128,0.004300799965858459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,32,64,0.0039594667653242745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,32,768,0.006621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,64,32,32,0.0040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,65536,5120,0.4201813379923503
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,65536,4096,0.33280000686645506
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,65536,3584,0.2994175910949707
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,65536,12288,1.015500831604004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,65536,2560,0.2390357335408529
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,65536,6144,0.5000874519348144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,65536,7168,0.6440618515014649
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,65536,2048,0.18629973729451496
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,65536,1536,0.14513492584228516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,65536,3072,0.2590378602345785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,65536,512,0.05382826725641886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,65536,768,0.07789226373036703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,65536,1024,0.0994645357131958
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,65536,8192,0.7350613276163738
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,65536,256,0.03331413269042969
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,65536,128,0.01996799906094869
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,65536,64,0.015052800377209982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,65536,32,0.014165332913398743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,65536,16384,1.3387775421142578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,16384,8192,0.17933653195699056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,16384,12288,0.2574336051940918
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,65536,10240,0.9151487986246745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,16384,6144,0.14909440676371258
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,16384,5120,0.11284480094909669
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,16384,7168,0.15571626027425128
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,16384,4096,0.09792853196461995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,16384,10240,0.21899946530659994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,16384,2560,0.06085973183314005
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,16384,3072,0.07150932947794596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,16384,1536,0.04140373468399048
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,16384,3584,0.08973653316497802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,16384,512,0.015291733543078103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,16384,1024,0.029422932863235475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,16384,768,0.0211626668771108
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,16384,2048,0.054271999994913736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,16384,16384,0.3798015912373861
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,16384,256,0.01027413308620453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,16384,64,0.006109866499900818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,16384,128,0.007099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,16384,32,0.0053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,12288,8192,0.1372159957885742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,12288,10240,0.17175893783569335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,12288,5120,0.08229546546936035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,12288,4096,0.07232853571573893
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,12288,6144,0.09717760086059571
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,12288,7168,0.1257472038269043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,12288,12288,0.2262357393900553
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,12288,16384,0.28869972229003904
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,12288,3584,0.059699201583862306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,12288,3072,0.0559445341428121
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,12288,2560,0.04747946659723918
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,12288,2048,0.036556800206502274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,12288,768,0.01532586713631948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,12288,1024,0.021913599967956544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,12288,512,0.012731732924779257
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,12288,1536,0.030856533845265703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,12288,64,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,12288,256,0.007816533247629803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,12288,128,0.005700266857941946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,12288,32,0.004744533201058706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,10240,12288,0.156876802444458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,10240,7168,0.0958463986714681
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,10240,10240,0.1335637410481771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,10240,8192,0.11212800343831379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,10240,6144,0.09065813223520915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,12288,65536,0.9816746393839517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,10240,16384,0.2179413318634033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,10240,3072,0.04795733292897542
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,10240,3584,0.05044906536738077
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,10240,4096,0.06191786527633667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,10240,5120,0.06970026493072509
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,10240,2560,0.040277334054311116
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,10240,1536,0.026077866554260254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,10240,2048,0.032631466786066696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,10240,128,0.0054954667886098225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,10240,256,0.0065536002318064375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,10240,768,0.014813866217931113
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,10240,1024,0.016588800152142844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,10240,512,0.011264000336329143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,10240,32,0.004300799965858459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,10240,64,0.005188266436258951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,8192,8192,0.09096533457438151
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,16384,65536,1.3381632486979167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,10240,65536,0.8562005360921224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,8192,7168,0.07778986295064291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,8192,10240,0.11362986564636231
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,8192,12288,0.13329066435496012
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,8192,6144,0.067413330078125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,8192,5120,0.06335146824518839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,8192,16384,0.1787903944651286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,8192,4096,0.05044906536738077
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,8192,2560,0.03211946686108907
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,8192,2048,0.02764799992243449
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,8192,3072,0.04031146764755249
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,8192,3584,0.04580693244934082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,8192,1536,0.020275199413299562
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,8192,768,0.01181013286113739
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,8192,256,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,8192,512,0.009284266829490661
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,8192,64,0.004744533201058706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,8192,128,0.005120000243186951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,8192,1024,0.015803733468055726
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,8192,32,0.0040277334551016486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,7168,16384,0.15612586339314777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,7168,7168,0.07079253196716309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,7168,8192,0.08314879735310873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,7168,10240,0.1016149361928304
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,7168,12288,0.118340269724528
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,7168,6144,0.0618837316830953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,8192,65536,0.6821887969970704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,7168,5120,0.056900266806284586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,7168,3584,0.04133546749750773
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,7168,2560,0.0289792001247406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,7168,2048,0.025224532683690386
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,7168,3072,0.03645439942677815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,7168,1536,0.019933867454528808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,7168,4096,0.04177920023600261
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,7168,1024,0.014711466431617738
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,7168,512,0.008772266904513042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,7168,256,0.006075733403364817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,7168,768,0.012049067020416259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,7168,128,0.005120000243186951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,7168,64,0.004983466863632202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,7168,32,0.003822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,6144,7168,0.06413653294245401
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,6144,8192,0.07591253121693929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,6144,10240,0.08994133472442627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,6144,12288,0.10318506558736165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,7168,65536,0.5806079864501953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,6144,5120,0.04427093267440796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,6144,16384,0.13755733172098797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,6144,6144,0.05604693492253622
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,6144,2560,0.02802346746126811
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,6144,4096,0.03904853264490764
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,6144,3584,0.03758080005645752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,6144,2048,0.02259626587231954
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,6144,3072,0.03317760030428569
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,6144,1536,0.01672533353169759
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,6144,1024,0.012663466731707254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,6144,768,0.009966933727264404
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,6144,512,0.008191999793052674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,6144,256,0.006178133189678192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,6144,128,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,6144,64,0.004198400179545084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,6144,32,0.003618133316437403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,5120,8192,0.05905066728591919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,6144,65536,0.4964351971944173
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,5120,16384,0.11373226642608643
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,5120,6144,0.043144532044728595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,5120,7168,0.0501418670018514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,5120,10240,0.0702122688293457
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,5120,12288,0.08809813658396402
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,5120,3584,0.027306665976842243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,5120,5120,0.03659093379974365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,5120,4096,0.033655468622843424
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,5120,2560,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,5120,3072,0.026077866554260254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,5120,1024,0.011502933502197266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,5120,2048,0.018261333306630455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,5120,1536,0.01495039959748586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,5120,768,0.009727999567985535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,5120,512,0.007236266632874806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,5120,64,0.004300799965858459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,5120,256,0.00576853354771932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,5120,128,0.004983466863632202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,5120,32,0.003618133316437403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,4096,12288,0.07410346666971843
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,4096,16384,0.0934229294459025
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,4096,10240,0.06557013193766276
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,4096,5120,0.031300266583760575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,4096,7168,0.04164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,4096,8192,0.048469332853953044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,4096,6144,0.036659200986226395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,4096,3584,0.025088000297546386
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,4096,4096,0.026828799645106
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,5120,65536,0.4145493189493815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,4096,3072,0.021026132504145305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,4096,1024,0.010308266679445902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,4096,768,0.008123733103275299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,4096,2560,0.01986560026804606
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,4096,2048,0.016145066420237223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,4096,1536,0.013482667009035745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,4096,512,0.006587733328342438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,4096,256,0.005666133264700572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,4096,128,0.004949333270390829
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,4096,64,0.0040277334551016486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,4096,32,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3584,8192,0.04870826800664266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3584,10240,0.056524801254272464
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3584,7168,0.038126933574676516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3584,4096,0.02290346622467041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3584,12288,0.06635520060857138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3584,5120,0.02874026695887248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3584,16384,0.08502613703409831
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3584,6144,0.03314346671104431
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3584,3584,0.02321066657702128
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,4096,65536,0.3359061241149902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3584,3072,0.01867093245188395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3584,2048,0.014677332838376364
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3584,1536,0.01225386659304301
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3584,768,0.0073728000124295544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3584,1024,0.009659733374913533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3584,2560,0.0159061332543691
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3584,512,0.006451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3584,256,0.005393066505591074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3584,128,0.00477866679430008
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3584,32,0.0034133332471052804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3584,64,0.003993600110212962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3072,8192,0.04392960071563721
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3072,10240,0.05147306521733602
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3072,7168,0.03891199827194214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3072,12288,0.06324906746546427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3072,5120,0.026589866479237872
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3072,16384,0.0718506654103597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3584,65536,0.2966869354248047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3072,4096,0.02249386707941691
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3072,3584,0.02106026609738668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3072,6144,0.034986666838328045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3072,2560,0.017339734236399333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3072,1024,0.008874666690826417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3072,3072,0.018158932526906334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3072,1536,0.012049067020416259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3072,2048,0.013312000036239623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3072,768,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3072,256,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3072,512,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3072,128,0.0044714664419492085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3072,64,0.003822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3072,32,0.003549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2560,12288,0.048230401674906415
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2560,8192,0.034679468472798666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2560,16384,0.06277120113372803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,3072,65536,0.2530303955078125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2560,10240,0.040550398826599124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2560,6144,0.02757973273595174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2560,7168,0.03430399894714355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2560,5120,0.02542933424313863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2560,3584,0.019387733936309815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2560,3072,0.017373865842819212
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2560,2560,0.01518933375676473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2560,1024,0.008055466910203297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2560,4096,0.019694934288660683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2560,1536,0.01088853379090627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2560,2048,0.011946666240692138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2560,768,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2560,512,0.006109866499900818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2560,128,0.004334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2560,256,0.005188266436258951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2560,65536,0.2155519962310791
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2560,32,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2560,64,0.007748266557852428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2048,10240,0.0339626669883728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2048,16384,0.05259946584701538
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2048,8192,0.02908160090446472
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2048,7168,0.036761601765950516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2048,12288,0.04031146764755249
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2048,5120,0.019899733861287437
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2048,6144,0.023108265797297158
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2048,4096,0.018500266472498576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2048,3584,0.01693013310432434
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2048,3072,0.014574933052062988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2048,2560,0.013619200388590495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2048,768,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2048,1024,0.007850666840871174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2048,2048,0.01160533328851064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2048,1536,0.009864532947540283
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2048,512,0.006109866499900818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2048,65536,0.17807359695434571
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2048,64,0.003549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2048,256,0.005188266436258951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2048,128,0.004198400179545084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,2048,32,0.0033791999022165934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1536,12288,0.03222186764081319
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1536,16384,0.03935573498408
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1536,8192,0.02321066657702128
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1536,10240,0.028910932938257854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1536,7168,0.020172800620396933
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1536,6144,0.01812480092048645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1536,5120,0.015803733468055726
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1536,4096,0.01532586713631948
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1536,3072,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1536,3584,0.014301866292953491
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1536,2560,0.011434666315714518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1536,2048,0.011502933502197266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1536,1024,0.007850666840871174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1536,1536,0.009591466188430786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1536,768,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1536,65536,0.1324031988779704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1536,512,0.006178133189678192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1536,256,0.005188266436258951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1536,128,0.0037205333511034647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1536,64,0.003549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1536,32,0.00314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1024,10240,0.020138667027155558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1024,16384,0.029422932863235475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1024,12288,0.02512213389078776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1024,7168,0.015359999736150107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1024,8192,0.016759467124938966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1024,6144,0.019421867529551187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1024,4096,0.011673600474993388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1024,5120,0.012834133704503379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1024,3584,0.011195733149846395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1024,3072,0.00993280013402303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1024,65536,0.09492479960123698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1024,1024,0.007816533247629803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1024,2560,0.009045333663622538
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1024,1536,0.009591466188430786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1024,768,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1024,2048,0.008123733103275299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1024,512,0.006007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1024,256,0.004676266511281332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1024,128,0.0034815999368826545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1024,64,0.0033791999022165934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,1024,32,0.0029696000119050344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,768,10240,0.01518933375676473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,768,16384,0.023483733336130776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,768,12288,0.01781760056813558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,768,8192,0.013550933202107748
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,768,6144,0.011878400047620138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,768,7168,0.01269760032494863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,768,5120,0.010956799983978272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,768,65536,0.07215786774953206
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,768,4096,0.009386666615804036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,768,3584,0.008874666690826417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,768,2560,0.008089600006739299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,768,2048,0.007816533247629803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,768,3072,0.008601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,768,1536,0.007406933108965556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,768,1024,0.006792533397674561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,768,512,0.005905066430568695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,768,768,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,768,256,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,768,64,0.0030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,768,128,0.0034133332471052804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,768,32,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,512,16384,0.016964266697565712
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,512,7168,0.010752000411351522
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,512,12288,0.013994666934013366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,512,8192,0.01129813293615977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,512,10240,0.01276586651802063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,512,6144,0.00955733358860016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,512,65536,0.05174613396326701
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,512,5120,0.008840533097585042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,512,3584,0.007748266557852428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,512,4096,0.008089600006739299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,512,2560,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,512,2048,0.006826666494210561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,512,3072,0.00747519979874293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,512,1536,0.006724266707897187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,512,1024,0.006382933259010315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,512,512,0.00631466656923294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,512,256,0.004334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,512,768,0.006826666494210561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,512,32,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,512,128,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,512,64,0.0040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,256,16384,0.011673600474993388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,256,65536,0.031197865804036457
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,256,12288,0.009864532947540283
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,256,10240,0.00897706647713979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,256,6144,0.007645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,256,5120,0.0073728000124295544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,256,7168,0.007850666840871174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,256,8192,0.008328533172607422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,256,3072,0.006894933183987935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,256,4096,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,256,3584,0.007645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,256,2560,0.00747519979874293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,256,2048,0.006451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,256,1536,0.0065536002318064375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,256,1024,0.006178133189678192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,256,512,0.006041599810123444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,256,128,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,256,256,0.004266666869322458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,256,768,0.006826666494210561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,256,64,0.0040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,256,32,0.004130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,128,16384,0.008840533097585042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,128,65536,0.017851734161376955
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,128,12288,0.008260266482830047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,128,10240,0.008021333316961924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,128,8192,0.007714133461316426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,128,6144,0.007406933108965556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,128,5120,0.0075434664885203045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,128,7168,0.007645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,128,4096,0.007099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,128,3072,0.006758399804433187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,128,3584,0.007133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,128,2560,0.006690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,128,2048,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,128,1536,0.005905066430568695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,128,1024,0.005427200098832448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,128,256,0.005085866649945577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,128,512,0.005870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,128,128,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,128,768,0.006792533397674561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,128,64,0.0039594667653242745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,128,32,0.004130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,64,16384,0.009147733449935913
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,64,65536,0.013653332988421122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,64,12288,0.008874666690826417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,64,8192,0.008430932958920796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,64,10240,0.008806399504343669
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,64,6144,0.0076117331782976794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,64,7168,0.008089600006739299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,64,5120,0.007441066702206929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,64,4096,0.007748266557852428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,64,3584,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,64,3072,0.006690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,64,2560,0.006826666494210561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,64,2048,0.006690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,64,1536,0.0056320001681645715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,64,512,0.00580266664425532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,64,768,0.006724266707897187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,64,128,0.004334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,64,256,0.004983466863632202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,64,64,0.0039594667653242745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,64,32,0.0040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,64,1024,0.0052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,32,65536,0.012219732999801636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,32,16384,0.009045333663622538
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,32,10240,0.008533333738644917
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,32,12288,0.008669867118199667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,32,8192,0.008191999793052674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,32,7168,0.00795306662718455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,32,6144,0.0075434664885203045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,32,5120,0.007406933108965556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,32,4096,0.0075434664885203045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,32,3072,0.0065536002318064375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,32,3584,0.0073728000124295544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,32,2048,0.006485333542029063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,32,2560,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,32,1536,0.0055637334783871974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,32,1024,0.005870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,32,256,0.004915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,32,512,0.005734399954477946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,32,128,0.004232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,32,768,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,32,64,0.003993600110212962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,48,32,32,0.0040277334551016486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,65536,4096,0.33187840779622396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,65536,6144,0.500053342183431
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,65536,10240,0.8210773468017578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,65536,3072,0.2808490753173828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,65536,5120,0.4203178723653157
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,65536,3584,0.29644800821940104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,65536,12288,0.9733119964599609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,65536,2560,0.23627093633015953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,65536,1024,0.09820160071055094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,65536,2048,0.18551467259724935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,65536,7168,0.5785941441853841
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,65536,128,0.019694934288660683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,65536,256,0.032051199674606325
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,65536,768,0.07492266496022543
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,65536,512,0.05529599984486898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,65536,8192,0.6624597549438477
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,65536,1536,0.14295040766398112
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,65536,64,0.01262933313846588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,65536,32,0.008840533097585042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,16384,6144,0.1386837323506673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,16384,12288,0.2579797267913818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,16384,16384,0.3365887959798177
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,16384,4096,0.10209279855092365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,16384,7168,0.16066560745239258
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,16384,8192,0.18408106168111166
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,16384,5120,0.11690666675567626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,16384,10240,0.24214186668395996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,16384,3584,0.09349119663238525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,16384,3072,0.07441066900889079
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,16384,1536,0.04358826478322347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,16384,512,0.018261333306630455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,16384,768,0.029320534070332843
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,16384,2048,0.054647465546925865
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,16384,1024,0.031573333342870075
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,16384,2560,0.06355626583099365
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,16384,256,0.010308266679445902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,16384,128,0.005973333120346069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,16384,64,0.005154133339722951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,16384,32,0.004095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,12288,8192,0.12919466495513915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,12288,10240,0.1734656016031901
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,12288,12288,0.20196693738301597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,12288,5120,0.08946346441904704
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,12288,4096,0.07219200134277344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,65536,16384,1.3223594665527343
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,12288,6144,0.09693866570790609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,12288,7168,0.1112063964207967
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,12288,3584,0.05911893447240194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,12288,2048,0.039082666238149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,12288,3072,0.051268267631530764
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,12288,16384,0.27149651845296224
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,12288,1024,0.021640533208847047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,12288,768,0.016793600718180337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,12288,2560,0.04696746667226155
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,12288,512,0.01276586651802063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,12288,1536,0.029934932788213093
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,12288,128,0.005358933409055074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,12288,64,0.004232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,12288,256,0.008703999718030294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,12288,32,0.0036864000062147772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,10240,12288,0.1693354606628418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,10240,10240,0.13322239716847736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,16384,65536,1.386359405517578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,10240,8192,0.11141119798024494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,10240,5120,0.07731200059254964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,10240,7168,0.09608533382415771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,10240,6144,0.09065813223520915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,10240,4096,0.06277120113372803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,10240,16384,0.22783999443054198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,10240,3072,0.047650134563446044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,10240,2048,0.03259733319282532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,10240,3584,0.05618346532185873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,10240,2560,0.040243200461069745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,10240,1536,0.024098134040832518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,10240,768,0.014745600024859109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,10240,1024,0.018363734086354576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,10240,256,0.0065194666385650635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,10240,512,0.010308266679445902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,10240,64,0.004095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,10240,128,0.005017599960168203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,10240,32,0.003583999971548716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,8192,7168,0.08577706813812255
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,8192,10240,0.11898880004882813
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,8192,12288,0.13503146171569824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,8192,8192,0.09881599744160971
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,8192,6144,0.07519573370615641
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,10240,65536,0.8093354543050131
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,8192,4096,0.055910400549570714
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,8192,5120,0.06802773475646973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,12288,65536,1.007479476928711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,8192,16384,0.18029227256774902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,8192,2048,0.027374933163324993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,8192,3072,0.040004265308380124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,8192,1536,0.02177706758181254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,8192,2560,0.03379199902216594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,8192,3584,0.04952746629714966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,8192,768,0.012800000111262002
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,8192,1024,0.0159061332543691
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,8192,512,0.009352533022562663
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,8192,256,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,8192,128,0.004846933484077454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,8192,64,0.003857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,8192,32,0.0034815999368826545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,7168,7168,0.0834559996922811
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,7168,8192,0.09611946741739909
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,7168,10240,0.11653119723002117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,7168,12288,0.12612266540527345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,7168,3072,0.03631786505381267
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,7168,3584,0.048810664812723795
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,7168,4096,0.05437440077463786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,7168,5120,0.06181546847025553
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,7168,6144,0.07219200134277344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,7168,2560,0.028330665826797486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,7168,1536,0.01952426632245382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,7168,2048,0.025088000297546386
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,7168,16384,0.15899306933085125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,7168,1024,0.014574933052062988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,7168,512,0.00955733358860016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,7168,256,0.005973333120346069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,7168,768,0.011980799833933513
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,7168,128,0.0045056000351905824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,7168,32,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,7168,64,0.003822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,6144,10240,0.08144213358561198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,8192,65536,0.6571690877278645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,6144,16384,0.13810346921284994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,6144,8192,0.0686079978942871
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,6144,6144,0.050995198885599766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,6144,12288,0.11881813208262126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,6144,7168,0.058641068140665685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,6144,4096,0.03904853264490764
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,6144,3584,0.034679468472798666
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,6144,5120,0.04734293222427368
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,6144,3072,0.02969599962234497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,6144,2048,0.020923733711242676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,6144,2560,0.02529279987017314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,6144,1536,0.016759467124938966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,6144,512,0.007884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,6144,1024,0.012663466731707254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,6144,768,0.014916266997655234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,6144,64,0.0037546666959921518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,6144,256,0.0056320001681645715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,6144,128,0.004676266511281332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,6144,32,0.003549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,7168,65536,0.5918037414550781
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,6144,65536,0.4902229309082031
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,5120,12288,0.08635733127593995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,5120,7168,0.054647465546925865
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,5120,8192,0.06242986520131429
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,5120,5120,0.04638719956080119
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,5120,10240,0.0731818675994873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,5120,6144,0.05239466826121012
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,5120,4096,0.03307519952456157
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,5120,16384,0.11209386984507244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,5120,3584,0.0289792001247406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,5120,3072,0.025463465849558515
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,5120,2048,0.018261333306630455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,5120,2560,0.020070399840672812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,5120,1536,0.01372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,5120,1024,0.01160533328851064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,5120,768,0.00897706647713979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,5120,512,0.006758399804433187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,5120,256,0.005393066505591074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,5120,64,0.003618133316437403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,5120,128,0.0046079998215039575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,5120,32,0.0035157332817713416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,4096,12288,0.07239680290222168
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,4096,8192,0.05085866848627726
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,5120,65536,0.4098047892252604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,4096,7168,0.04304213523864746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,4096,6144,0.038365864753723146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,4096,10240,0.05833386580149332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,4096,16384,0.09314986864725748
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,4096,5120,0.035259731610616046
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,4096,4096,0.027170133590698243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,4096,3584,0.024985599517822265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,4096,768,0.007987200220425924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,4096,1024,0.01027413308620453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,4096,1536,0.012014933427174886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,4096,2560,0.01812480092048645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,4096,2048,0.015803733468055726
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,4096,3072,0.02177706758181254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,4096,256,0.005256533126036326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,4096,512,0.006212266782919565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,4096,128,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,4096,32,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,4096,64,0.003618133316437403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,4096,65536,0.33508691787719724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3584,8192,0.0457045316696167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3584,12288,0.06393173138300577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3584,10240,0.05522773265838623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3584,5120,0.034713598092397054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3584,16384,0.08628906408945719
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3584,4096,0.024337067206700643
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3584,6144,0.04184746742248535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3584,3584,0.022732800245285033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3584,7168,0.04201813141504924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3584,3072,0.018397865692774455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3584,2560,0.015735466281572977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3584,2048,0.01460906664530436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3584,768,0.007133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3584,1536,0.012288000186284382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3584,1024,0.00976213316122691
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3584,512,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3584,256,0.0053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3584,128,0.004334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3584,32,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3584,64,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3072,8192,0.037887998421986896
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3072,7168,0.034406399726867674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3072,10240,0.04580693244934082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3584,65536,0.29784746170043946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3072,12288,0.05761706829071045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3072,16384,0.07669760386149088
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3072,6144,0.031129600604375203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3072,4096,0.020514132579167683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3072,5120,0.026999467611312868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3072,3584,0.020889600118001304
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3072,2560,0.01628159979979197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3072,3072,0.018636800845464072
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3072,2048,0.014301866292953491
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3072,1536,0.011980799833933513
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3072,1024,0.00890880028406779
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3072,768,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3072,512,0.006109866499900818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3072,256,0.005154133339722951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3072,128,0.0040277334551016486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3072,65536,0.25337173144022623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3072,32,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,3072,64,0.0037205333511034647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2560,12288,0.045943466822306316
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2560,16384,0.059494400024414064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2560,7168,0.0318122665087382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2560,6144,0.02723840077718099
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2560,5120,0.022971733411153158
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2560,8192,0.03266560037930806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2560,10240,0.039901868502298994
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2560,4096,0.020514132579167683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2560,3584,0.019421867529551187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2560,3072,0.015940266847610473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2560,2048,0.011980799833933513
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2560,2560,0.013994666934013366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2560,1536,0.010786133011182149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2560,1024,0.00795306662718455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2560,512,0.006109866499900818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2560,768,0.006963199873765309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2560,65536,0.21265066464742027
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2560,256,0.005154133339722951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2560,128,0.003857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2560,64,0.003583999971548716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2560,32,0.0032085334261258446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2048,10240,0.03628373146057129
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2048,12288,0.03884373505910237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2048,8192,0.028330665826797486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2048,16384,0.04870826800664266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2048,6144,0.022971733411153158
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2048,5120,0.019285333156585694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2048,7168,0.02573653260866801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2048,4096,0.016110933820406594
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2048,3072,0.014028799533843995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2048,3584,0.016247466206550598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2048,2560,0.01443839967250824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2048,1536,0.009796266754468281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2048,1024,0.007850666840871174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2048,2048,0.011571199695269267
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2048,768,0.006963199873765309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2048,128,0.003857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2048,256,0.004676266511281332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2048,512,0.006178133189678192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2048,65536,0.17640105883280438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2048,64,0.0033791999022165934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,2048,32,0.0031061333914597826
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1536,16384,0.039082666238149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1536,10240,0.026146133740743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1536,12288,0.030037333567937214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1536,8192,0.021606399615605672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1536,7168,0.02044586737950643
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1536,4096,0.013414399822553
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1536,3072,0.012390399972597759
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1536,3584,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1536,2560,0.010717866818110149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1536,6144,0.017032533884048462
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1536,5120,0.01495039959748586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1536,2048,0.009181867043177288
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1536,1536,0.008260266482830047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1536,1024,0.007441066702206929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1536,65536,0.13482666015625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1536,768,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1536,256,0.0045056000351905824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1536,128,0.00365226666132609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1536,512,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1536,32,0.004198400179545084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1536,64,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1024,12288,0.022835199038187662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1024,7168,0.015223466356595359
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1024,10240,0.019387733936309815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1024,8192,0.016383999586105348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1024,16384,0.03054933349291484
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1024,6144,0.01378986636797587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1024,5120,0.012322133779525757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1024,3584,0.010547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1024,65536,0.092740265528361
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1024,4096,0.011195733149846395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1024,2560,0.008703999718030294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1024,3072,0.009386666615804036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1024,2048,0.007918933530648549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1024,1536,0.007441066702206929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1024,1024,0.006724266707897187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1024,512,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1024,768,0.006792533397674561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1024,128,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1024,64,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1024,32,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,1024,256,0.004369066655635833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,768,16384,0.021913599967956544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,768,12288,0.01699840029080709
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,768,10240,0.014984533190727234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,768,65536,0.07079253196716309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,768,7168,0.012390399972597759
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,768,6144,0.011502933502197266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,768,8192,0.013243732849756875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,768,5120,0.010820266604423524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,768,4096,0.009113599856694538
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,768,3584,0.008601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,768,3072,0.008089600006739299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,768,2560,0.007850666840871174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,768,2048,0.007338666419188182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,768,1536,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,768,1024,0.0065194666385650635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,768,512,0.005870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,768,768,0.006758399804433187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,768,256,0.004334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,768,128,0.0033791999022165934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,768,64,0.004095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,768,32,0.004130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,512,16384,0.015735466281572977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,512,8192,0.01129813293615977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,512,12288,0.013926399747530618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,512,10240,0.012288000186284382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,512,6144,0.009489066402117411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,512,7168,0.010444800059000652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,512,65536,0.05253119866053263
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,512,5120,0.008669867118199667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,512,4096,0.007850666840871174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,512,3584,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,512,3072,0.007338666419188182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,512,2048,0.006724266707897187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,512,2560,0.007099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,512,1536,0.0065536002318064375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,512,1024,0.00628053347269694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,512,768,0.006758399804433187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,512,512,0.005870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,512,256,0.004232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,512,128,0.0032768001159032187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,512,64,0.0040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,512,32,0.004130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,256,16384,0.011434666315714518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,256,12288,0.009659733374913533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,256,10240,0.008772266904513042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,256,65536,0.030856533845265703
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,256,8192,0.008055466910203297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,256,6144,0.0076799998680750535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,256,7168,0.007748266557852428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,256,4096,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,256,3584,0.0076117331782976794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,256,5120,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,256,3072,0.006690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,256,2560,0.00730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,256,2048,0.006451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,256,1536,0.006451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,256,1024,0.005529599885145823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,256,768,0.006724266707897187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,256,512,0.005905066430568695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,256,128,0.004334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,256,256,0.004198400179545084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,256,64,0.0040277334551016486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,256,32,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,128,16384,0.008738133311271667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,128,65536,0.017066667477289833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,128,12288,0.008123733103275299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,128,10240,0.007918933530648549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,128,8192,0.0076117331782976794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,128,5120,0.007441066702206929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,128,6144,0.007441066702206929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,128,4096,0.007099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,128,3584,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,128,7168,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,128,3072,0.006690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,128,2560,0.006690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,128,2048,0.006792533397674561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,128,1536,0.0055978665749231975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,128,1024,0.0052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,128,768,0.006724266707897187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,128,512,0.005870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,128,256,0.005017599960168203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,128,128,0.004266666869322458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,128,64,0.003993600110212962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,128,32,0.0040277334551016486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,64,16384,0.009011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,64,65536,0.013619200388590495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,64,12288,0.008635733524958293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,64,10240,0.008703999718030294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,64,7168,0.00795306662718455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,64,6144,0.0076117331782976794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,64,5120,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,64,8192,0.008260266482830047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,64,3584,0.00747519979874293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,64,2560,0.0065536002318064375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,64,4096,0.0076799998680750535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,64,3072,0.0065194666385650635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,64,2048,0.006587733328342438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,64,1536,0.005529599885145823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,64,1024,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,64,512,0.00580266664425532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,64,256,0.004983466863632202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,64,64,0.0039594667653242745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,64,768,0.006724266707897187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,64,128,0.004266666869322458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,64,32,0.003993600110212962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,32,16384,0.00890880028406779
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,32,65536,0.012219732999801636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,32,10240,0.008669867118199667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,32,12288,0.008601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,32,8192,0.008157866696516674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,32,6144,0.00747519979874293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,32,7168,0.007884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,32,5120,0.007167999943097432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,32,4096,0.007406933108965556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,32,3584,0.007202133536338806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,32,2560,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,32,3072,0.006382933259010315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,32,2048,0.006417066852251689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,32,1536,0.006075733403364817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,32,768,0.0065194666385650635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,32,1024,0.005734399954477946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,32,512,0.005700266857941946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,32,64,0.0038912000755469004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,32,256,0.004881066580613455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,32,128,0.004232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,32,32,32,0.003993600110212962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,65536,5120,0.4210346539815267
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,65536,7168,0.586240005493164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,65536,4096,0.343176523844401
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,65536,10240,0.8167423884073892
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,65536,3072,0.2874709447224935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,65536,3584,0.31105705897013347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,65536,6144,0.5047978719075521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,65536,1536,0.14605654080708821
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,65536,2048,0.19206825892130536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,65536,1024,0.1033898671468099
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,65536,512,0.05321386655171713
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,65536,2560,0.22835200627644858
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,65536,768,0.07430826822916667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,65536,256,0.037478399276733396
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,65536,64,0.009352533022562663
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,65536,8192,0.6654293060302734
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,65536,128,0.015530666708946228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,65536,32,0.007167999943097432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,65536,16384,1.2920832316080728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,16384,8192,0.17762986818949383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,16384,6144,0.14714879989624025
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,16384,12288,0.2506410598754883
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,16384,16384,0.3264512062072754
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,16384,3584,0.09209173520406087
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,16384,7168,0.156330664952596
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,16384,10240,0.23070720036824546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,65536,12288,0.9687039693196615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,16384,5120,0.11615573565165202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,16384,4096,0.09605120023091634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,16384,3072,0.08120319843292237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,16384,768,0.023449599742889404
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,16384,512,0.015428266922632852
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,16384,2560,0.06208853324254354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,16384,1024,0.031231999397277832
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,16384,1536,0.043144532044728595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,16384,2048,0.05512533187866211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,16384,256,0.010342400272687275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,16384,128,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,16384,64,0.004676266511281332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,16384,32,0.004095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,12288,6144,0.1033898671468099
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,12288,7168,0.11844267050425213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,12288,10240,0.16233812967936198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,12288,8192,0.13516799608866376
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,12288,5120,0.08915627002716064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,12288,4096,0.06536533435185751
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,12288,3584,0.06365866661071777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,12288,3072,0.055569068590799964
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,12288,2560,0.04741119941075643
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,12288,2048,0.03696639935175578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,12288,1024,0.02259626587231954
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,12288,1536,0.031300266583760575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,12288,12288,0.2061994711558024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,12288,768,0.020036266247431437
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,12288,256,0.007850666840871174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,12288,512,0.01378986636797587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,12288,128,0.00481279989083608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,12288,16384,0.24886612892150878
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,12288,32,0.003788800040880839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,12288,64,0.0040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,10240,8192,0.11427840391794841
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,10240,12288,0.1586517333984375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,10240,16384,0.20660907427469888
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,10240,6144,0.09011200269063314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,10240,5120,0.07475199699401855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,10240,4096,0.05700266758600871
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,10240,7168,0.1008639971415202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,10240,10240,0.13619200388590497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,10240,3584,0.05505706469217936
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,10240,2560,0.03833173513412476
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,10240,3072,0.048059733708699544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,10240,1024,0.01976319948832194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,10240,768,0.017749333381652833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,10240,2048,0.034030934174855546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,10240,256,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,10240,1536,0.02744320034980774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,10240,512,0.013755733768145243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,10240,32,0.003788800040880839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,10240,128,0.0045056000351905824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,10240,64,0.003925333420435587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,8192,10240,0.10970453421274821
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,8192,12288,0.13038933277130127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,8192,16384,0.17254400253295898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,8192,6144,0.07673172950744629
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,8192,7168,0.08147626717885335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,8192,5120,0.06557013193766276
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,10240,65536,0.7678976058959961
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,8192,8192,0.09171626567840577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,8192,4096,0.05208746592203776
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,8192,3584,0.0425983985265096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,8192,1536,0.022664533058802287
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,8192,2560,0.034884266058603924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,16384,65536,1.2265813191731771
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,8192,1024,0.016076800227165223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,8192,3072,0.03809279998143514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,8192,2048,0.029183999697367353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,8192,768,0.016759467124938966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,8192,512,0.012117333213488261
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,8192,128,0.004232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,12288,65536,0.9249792098999023
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,8192,256,0.006792533397674561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,8192,64,0.0040277334551016486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,8192,32,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,7168,7168,0.0718506654103597
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,7168,12288,0.11499520142873128
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,7168,16384,0.15018666585286458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,7168,6144,0.06311253309249878
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,7168,10240,0.10775893529256184
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,7168,8192,0.08219306468963623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,7168,4096,0.04358826478322347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,7168,5120,0.05376000006993612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,7168,3584,0.03846826553344727
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,7168,1536,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,7168,3072,0.03679573138554891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,7168,2048,0.024337067206700643
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,7168,2560,0.029320534070332843
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,7168,1024,0.015052800377209982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,7168,768,0.01699840029080709
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,7168,512,0.01153706709543864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,7168,256,0.0065194666385650635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,7168,128,0.004130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,8192,65536,0.6227285385131835
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,7168,64,0.003583999971548716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,7168,32,0.003549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,6144,6144,0.0509610652923584
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,6144,16384,0.12827306588490803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,7168,65536,0.5595136006673177
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,6144,7168,0.05816320180892944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,6144,8192,0.07598079840342203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,6144,10240,0.0815445343653361
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,6144,12288,0.1001471996307373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,6144,3584,0.03399680058161418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,6144,5120,0.043315200010935466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,6144,4096,0.03570346832275391
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,6144,3072,0.029764266808827718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,6144,1536,0.016657066345214844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,6144,2560,0.023176532983779908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,6144,2048,0.020957867304484047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,6144,1024,0.01160533328851064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,6144,768,0.010205866893132527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,6144,512,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,6144,256,0.005700266857941946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,6144,128,0.0040277334551016486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,6144,64,0.0035157332817713416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,6144,32,0.0032426667710145317
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,5120,12288,0.0807253360748291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,5120,16384,0.10926079750061035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,6144,65536,0.4710399945576985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,5120,6144,0.04222293297449748
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,5120,7168,0.04918613433837891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,5120,8192,0.0579584002494812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,5120,10240,0.0686079978942871
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,5120,5120,0.03573760191599528
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,5120,3584,0.0289792001247406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,5120,4096,0.03266560037930806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,5120,3072,0.02525866627693176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,5120,2048,0.017988266547520955
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,5120,2560,0.02174293398857117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,5120,1536,0.014745600024859109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,5120,1024,0.01027413308620453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,5120,768,0.014062933127085366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,5120,256,0.005051733553409576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,5120,512,0.011127466956774395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,5120,128,0.004369066655635833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,5120,64,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,5120,32,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,5120,65536,0.39724372227986654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,4096,10240,0.055637331803639736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,4096,6144,0.03563520113627116
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,4096,7168,0.040516265233357746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,4096,8192,0.04713813463846843
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,4096,16384,0.08878080050150552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,4096,5120,0.03310933311780294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,4096,12288,0.06546773513158163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,4096,4096,0.026589866479237872
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,4096,2560,0.019080533583958944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,4096,3584,0.022528000672658286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,4096,3072,0.021811199188232423
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,4096,2048,0.014404267072677612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,4096,1536,0.011946666240692138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,4096,1024,0.01027413308620453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,4096,768,0.013346133629480996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,4096,512,0.010752000411351522
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,4096,256,0.004881066580613455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,4096,128,0.004300799965858459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,4096,32,0.0034133332471052804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,4096,64,0.0035157332817713416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,4096,65536,0.31802028020222983
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3584,7168,0.04761600097020467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3584,6144,0.04710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3584,5120,0.03932160139083862
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3584,16384,0.08065706888834635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3584,8192,0.05198506514231364
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3584,4096,0.03092479904492696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3584,3584,0.029730133215586346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3584,10240,0.051950931549072266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3584,12288,0.06004053354263306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3584,3072,0.02624853253364563
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3584,1536,0.012185600399971009
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3584,1024,0.00976213316122691
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3584,2560,0.024541866779327393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3584,2048,0.02109439969062805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3584,768,0.012049067020416259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3584,512,0.0106495996316274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3584,128,0.004266666869322458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3584,256,0.004881066580613455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3584,64,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3584,32,0.0034133332471052804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3072,7168,0.03314346671104431
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3072,8192,0.037205334504445395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3072,10240,0.04485119978586833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3072,6144,0.02986666758855184
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3584,65536,0.28166828155517576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3072,12288,0.05222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3072,16384,0.067686398824056
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3072,5120,0.025804799795150758
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3072,3584,0.024678399165471397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3072,4096,0.0237226665019989
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3072,3072,0.02174293398857117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3072,1024,0.00846506655216217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3072,768,0.01088853379090627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3072,2048,0.01791999936103821
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3072,2560,0.020138667027155558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3072,1536,0.011400533715883891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3072,512,0.00962559978167216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3072,64,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3072,128,0.003788800040880839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3072,256,0.004710400104522705
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3072,32,0.0032085334261258446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2560,12288,0.0452949325243632
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2560,8192,0.031778132915496825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2560,10240,0.03894613186518352
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,3072,65536,0.2448725382486979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2560,16384,0.05761706829071045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2560,7168,0.0289792001247406
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2560,5120,0.024098134040832518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2560,6144,0.026282666126887004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2560,4096,0.02065066695213318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2560,3584,0.023586134115854897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2560,3072,0.021538132429122926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2560,1536,0.010820266604423524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2560,2560,0.019080533583958944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2560,2048,0.016827734311421712
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2560,768,0.010683733224868774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2560,512,0.009454933802286784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2560,1024,0.007748266557852428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2560,128,0.003618133316437403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2560,256,0.0044714664419492085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2560,64,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2560,32,0.0031744000812371576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2560,65536,0.20968106587727867
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2048,16384,0.05007359981536865
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2048,10240,0.03232426643371582
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2048,12288,0.037614933649698895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2048,8192,0.02775040070215861
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2048,7168,0.02505386670430501
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2048,6144,0.022357332706451415
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2048,4096,0.01771519978841146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2048,5120,0.02106026609738668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2048,3584,0.016520532965660095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2048,3072,0.014847999811172486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2048,2560,0.012492799758911132
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2048,1024,0.007816533247629803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2048,2048,0.011332266529401143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2048,65536,0.1726805369059245
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2048,768,0.009727999567985535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2048,512,0.008874666690826417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2048,1536,0.009693866968154908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2048,64,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2048,256,0.004266666869322458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2048,128,0.003549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,2048,32,0.0030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1536,12288,0.029764266808827718
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1536,8192,0.021230934063593547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1536,16384,0.03829760154088338
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1536,10240,0.02542933424313863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1536,7168,0.01843199928601583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1536,6144,0.016793600718180337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1536,5120,0.014745600024859109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1536,4096,0.014165332913398743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1536,65536,0.13540693918863933
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1536,3584,0.012219732999801636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1536,3072,0.011332266529401143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1536,2048,0.011776000261306763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1536,1536,0.009489066402117411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1536,2560,0.010854400197664897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1536,1024,0.007748266557852428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1536,256,0.004232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1536,512,0.005700266857941946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1536,768,0.006929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1536,128,0.0033791999022165934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1536,64,0.0030037333567937215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1536,32,0.0029696000119050344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1024,16384,0.027374933163324993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1024,12288,0.02177706758181254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1024,8192,0.01628159979979197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1024,10240,0.018636800845464072
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1024,7168,0.014677332838376364
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1024,6144,0.013516799608866373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1024,5120,0.012356266379356384
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1024,3584,0.010547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1024,4096,0.010990933577219645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1024,2560,0.010922666390736897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1024,3072,0.012458667159080505
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1024,65536,0.08881493409474692
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1024,2048,0.009591466188430786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1024,1536,0.008260266482830047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1024,1024,0.007577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1024,768,0.006690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1024,256,0.0040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1024,512,0.005427200098832448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1024,128,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1024,32,0.0030720000465710956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,1024,64,0.00314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,768,12288,0.019319466749827065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,768,10240,0.017885865767796834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,768,16384,0.02321066657702128
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,768,7168,0.015018666783968607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,768,8192,0.015530666708946228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,768,6144,0.014301866292953491
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,768,4096,0.012014933427174886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,768,65536,0.06908586819966635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,768,5120,0.013482667009035745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,768,3584,0.010854400197664897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,768,3072,0.009898666540781658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,768,2560,0.010308266679445902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,768,2048,0.007133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,768,1536,0.007782400151093801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,768,1024,0.0065536002318064375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,768,768,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,768,256,0.003993600110212962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,768,64,0.0030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,768,128,0.0034815999368826545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,768,32,0.0030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,768,512,0.005427200098832448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,512,16384,0.015735466281572977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,512,10240,0.01225386659304301
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,512,12288,0.013482667009035745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,512,8192,0.01109333336353302
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,512,7168,0.010513066252072652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,512,6144,0.009181867043177288
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,512,65536,0.051336534818013514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,512,4096,0.007987200220425924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,512,3584,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,512,5120,0.008430932958920796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,512,3072,0.00846506655216217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,512,2560,0.007884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,512,2048,0.00730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,512,1536,0.006929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,512,768,0.006621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,512,1024,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,512,512,0.005973333120346069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,512,128,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,512,256,0.0039594667653242745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,512,64,0.0030037333567937215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,512,32,0.0030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,256,16384,0.011366400122642516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,256,12288,0.00955733358860016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,256,65536,0.02792106668154399
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,256,10240,0.008669867118199667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,256,8192,0.007748266557852428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,256,6144,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,256,5120,0.007099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,256,7168,0.007577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,256,4096,0.006894933183987935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,256,3584,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,256,3072,0.006587733328342438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,256,2560,0.007202133536338806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,256,2048,0.006382933259010315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,256,1024,0.005973333120346069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,256,512,0.005700266857941946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,256,1536,0.006075733403364817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,256,768,0.006621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,256,256,0.0044714664419492085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,256,128,0.0034133332471052804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,256,64,0.0029696000119050344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,256,32,0.0029696000119050344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,128,16384,0.009693866968154908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,128,12288,0.008635733524958293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,128,65536,0.016759467124938966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,128,6144,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,128,8192,0.0076799998680750535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,128,7168,0.00730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,128,10240,0.008157866696516674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,128,5120,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,128,4096,0.006792533397674561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,128,3584,0.006690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,128,3072,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,128,2560,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,128,2048,0.006621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,128,1536,0.006007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,128,768,0.006587733328342438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,128,512,0.0055978665749231975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,128,256,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,128,128,0.0033791999022165934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,128,64,0.0029696000119050344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,128,1024,0.005905066430568695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,128,32,0.00290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,64,65536,0.01372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,64,16384,0.008669867118199667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,64,12288,0.008226133386294047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,64,8192,0.0075434664885203045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,64,7168,0.007645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,64,10240,0.007884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,64,5120,0.007577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,64,4096,0.006382933259010315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,64,6144,0.00730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,64,3584,0.006382933259010315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,64,3072,0.006109866499900818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,64,2560,0.005905066430568695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,64,2048,0.006417066852251689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,64,1536,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,64,1024,0.00576853354771932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,64,768,0.00580266664425532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,64,128,0.0034815999368826545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,64,256,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,64,512,0.005529599885145823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,64,32,0.002935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,64,64,0.0029696000119050344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,32,16384,0.00846506655216217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,32,65536,0.01160533328851064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,32,12288,0.007987200220425924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,32,10240,0.007748266557852428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,32,8192,0.007406933108965556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,32,6144,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,32,5120,0.007099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,32,4096,0.006178133189678192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,32,3072,0.006485333542029063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,32,3584,0.006621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,32,7168,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,32,2560,0.00628053347269694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,32,2048,0.00631466656923294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,32,1536,0.006109866499900818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,32,1024,0.0055978665749231975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,32,768,0.005666133264700572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,32,64,0.0030037333567937215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,32,256,0.004334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,32,512,0.005427200098832448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,32,128,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,16,32,32,0.00290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,65536,5120,0.41301333109537763
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,65536,7168,0.572927983601888
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,65536,8192,0.6586026509602865
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,65536,4096,0.33576958974202475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,65536,3584,0.30726826985677086
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,65536,2560,0.23968426386515299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,65536,3072,0.28197546005249025
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,65536,6144,0.4957866668701172
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,65536,1536,0.14462292989095052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,65536,768,0.07376213073730468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,65536,1024,0.10263893604278565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,65536,256,0.028194133440653486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,65536,64,0.009079466263453167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,65536,128,0.013619200388590495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,65536,32,0.0076799998680750535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,65536,2048,0.184115203221639
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,65536,512,0.05277013381322225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,16384,12288,0.24466773668924965
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,65536,16384,1.275153096516927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,16384,16384,0.3214336077372233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,65536,10240,0.8035327911376953
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,16384,8192,0.17363626162211102
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,16384,10240,0.20807679494222006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,16384,7168,0.15264426867167155
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,16384,5120,0.1237674633661906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,16384,6144,0.1454421361287435
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,65536,12288,0.9560064315795899
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,16384,4096,0.09980586369832357
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,16384,2560,0.06683306694030762
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,16384,3584,0.0903167963027954
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,16384,2048,0.05492053429285685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,16384,1024,0.03126613299051921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,16384,3072,0.07959893544514975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,16384,768,0.021401600042978922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,16384,1536,0.040209066867828366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,16384,128,0.005120000243186951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,16384,256,0.010001066327095031
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,16384,512,0.016383999586105348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,16384,64,0.004266666869322458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,16384,32,0.0037546666959921518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,12288,8192,0.13226666450500488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,12288,12288,0.18766506512959796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,12288,6144,0.10089813073476155
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,12288,7168,0.11625813643137614
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,12288,16384,0.24463359514872232
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,12288,10240,0.15810559590657552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,12288,5120,0.0869376023610433
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,12288,4096,0.07065599759419759
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,12288,2560,0.04730879863103231
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,12288,1536,0.031436800956726074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,12288,1024,0.022459733486175536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,12288,3072,0.050756267706553136
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,12288,3584,0.0579584002494812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,12288,2048,0.039185067017873124
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,12288,768,0.018500266472498576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,12288,256,0.007645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,12288,512,0.014404267072677612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,16384,65536,1.2332714080810547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,12288,128,0.0046079998215039575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,12288,64,0.003993600110212962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,12288,32,0.0034815999368826545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,10240,10240,0.14626132647196452
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,10240,12288,0.15796906153361004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,10240,16384,0.21039786338806152
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,10240,6144,0.0893951972325643
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,10240,7168,0.09888426462809244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,10240,8192,0.11281066735585529
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,10240,4096,0.0609279990196228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,10240,5120,0.07608319918314616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,10240,3584,0.05461333195368449
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,10240,1024,0.018773333231608073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,10240,2048,0.03160746693611145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,10240,1536,0.027340799570083618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,10240,2560,0.040994131565093996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,10240,3072,0.04778666496276855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,10240,768,0.018705066045125326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,10240,512,0.013004799683888754
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,10240,128,0.0044714664419492085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,10240,256,0.006894933183987935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,10240,32,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,10240,64,0.003857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,8192,16384,0.1672533353169759
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,8192,12288,0.12830719947814942
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,8192,7168,0.07987199624379476
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,8192,8192,0.09130667050679525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,8192,10240,0.1101482629776001
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,8192,6144,0.06966613133748373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,8192,5120,0.065228799978892
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,8192,2560,0.032358400026957196
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,8192,2048,0.028910932938257854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,12288,65536,0.9735509236653647
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,8192,3072,0.03775146802266439
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,8192,4096,0.0474453330039978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,8192,3584,0.046284798781077066
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,10240,65536,0.7633578618367513
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,8192,1536,0.020957867304484047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,8192,1024,0.015940266847610473
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,8192,768,0.017578667402267455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,8192,512,0.012049067020416259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,8192,256,0.005427200098832448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,8192,128,0.004198400179545084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,8192,32,0.0032426667710145317
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,8192,64,0.0036864000062147772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,7168,7168,0.0723626693089803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,7168,6144,0.06198613246281942
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,8192,65536,0.620680554707845
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,7168,8192,0.0818176031112671
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,7168,10240,0.0974506696065267
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,7168,5120,0.05358933210372925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,7168,12288,0.12079786459604899
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,7168,4096,0.043622398376464845
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,7168,3584,0.04147200187047322
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,7168,16384,0.15854934056599934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,7168,2048,0.025838933388392132
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,7168,2560,0.03099306623140971
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,7168,1536,0.01853440006573995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,7168,3072,0.036659200986226395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,7168,1024,0.015359999736150107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,7168,768,0.015974400440851848
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,7168,512,0.011434666315714518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,7168,256,0.0065194666385650635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,7168,128,0.0040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,7168,64,0.0034815999368826545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,7168,32,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,6144,10240,0.0807253360748291
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,6144,16384,0.12728319962819418
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,6144,6144,0.05027840137481689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,6144,7168,0.05775359869003296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,6144,8192,0.06853973070780436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,6144,5120,0.04751360019048055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,6144,12288,0.09888426462809244
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,7168,65536,0.5535743713378907
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,6144,4096,0.035327998797098796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,6144,2048,0.020718934138615926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,6144,1024,0.01262933313846588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,6144,2560,0.024985599517822265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,6144,3072,0.032358400026957196
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,6144,1536,0.015121066570281982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,6144,768,0.010205866893132527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,6144,3584,0.03092479904492696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,6144,512,0.007714133461316426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,6144,256,0.0056320001681645715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,6144,128,0.003993600110212962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,6144,64,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,6144,32,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,5120,12288,0.08045226732889811
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,5120,10240,0.06813013553619385
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,5120,6144,0.04194986820220947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,6144,65536,0.4670122782389323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,5120,16384,0.1204906702041626
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,5120,4096,0.03089066743850708
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,5120,7168,0.048742401599884036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,5120,5120,0.039082666238149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,5120,8192,0.05751466751098633
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,5120,2048,0.016452266772588094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,5120,1024,0.011264000336329143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,5120,3072,0.024985599517822265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,5120,2560,0.021708800395329794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,5120,1536,0.014745600024859109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,5120,768,0.01443839967250824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,5120,3584,0.02621440092722575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,5120,512,0.010820266604423524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,5120,256,0.005051733553409576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,5120,128,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,5120,32,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,5120,64,0.0034815999368826545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,4096,10240,0.05502293507258097
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,4096,12288,0.06533120075861612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,5120,65536,0.39123627344767253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,4096,8192,0.04696746667226155
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,4096,16384,0.09816746711730957
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,4096,6144,0.03502080043156942
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,4096,7168,0.040243200461069745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,4096,5120,0.032767999172210696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,4096,4096,0.024098134040832518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,4096,3584,0.024405332406361897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,4096,3072,0.021504000822703043
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,4096,1024,0.01027413308620453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,4096,768,0.012219732999801636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,4096,2560,0.018636800845464072
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,4096,256,0.004881066580613455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,4096,512,0.010615467031796774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,4096,2048,0.015701333681742348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,4096,1536,0.013038933277130127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,4096,128,0.004232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,4096,64,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,4096,32,0.0032768001159032187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3584,8192,0.05140479803085327
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3584,16384,0.07990612983703613
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3584,7168,0.048571733633677165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3584,10240,0.050756267706553136
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3584,12288,0.06017706791559855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,4096,65536,0.3147775967915853
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3584,5120,0.03642026583353679
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3584,3584,0.03109546701113383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3584,4096,0.03051519989967346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3584,6144,0.04154026508331299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3584,1536,0.012185600399971009
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3584,3072,0.027818665901819868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3584,2560,0.03126613299051921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3584,1024,0.009727999567985535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3584,768,0.011844266454378765
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3584,2048,0.021026132504145305
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3584,512,0.010547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3584,256,0.004846933484077454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3584,32,0.0032426667710145317
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3584,128,0.004232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3584,64,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3584,65536,0.28419411977132164
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3072,8192,0.03686399857203166
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3072,10240,0.04413439830144246
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3072,5120,0.027511467536290485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3072,6144,0.029149866104125975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3072,16384,0.06676479975382486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3072,7168,0.03269973397254944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3072,12288,0.05188266833623251
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3072,4096,0.023552000522613525
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3072,3584,0.024473599592844644
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3072,2048,0.01771519978841146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3072,2560,0.01952426632245382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3072,1536,0.011400533715883891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3072,1024,0.008328533172607422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3072,3072,0.020753065745035805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3072,768,0.010786133011182149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3072,512,0.00962559978167216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3072,256,0.0045056000351905824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3072,128,0.003788800040880839
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3072,64,0.0032426667710145317
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3072,65536,0.24391679763793944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,3072,32,0.0033791999022165934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2560,12288,0.0444757342338562
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2560,16384,0.05748053391774496
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2560,6144,0.027818665901819868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2560,5120,0.02198186715443929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2560,8192,0.03136853377024333
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2560,4096,0.02068480054537455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2560,7168,0.02836479942003886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2560,3584,0.023415466149648033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2560,10240,0.03775146802266439
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2560,3072,0.020138667027155558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2560,1536,0.010137599706649781
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2560,2560,0.01802240014076233
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2560,2048,0.016964266697565712
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2560,1024,0.007816533247629803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2560,768,0.0105813334385554
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2560,512,0.009215999643007915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2560,65536,0.2053119977315267
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2560,256,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2560,128,0.003583999971548716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2560,64,0.0032085334261258446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2560,32,0.0031744000812371576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2048,10240,0.03362133502960205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2048,8192,0.027374933163324993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2048,16384,0.04922026793162028
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2048,7168,0.0233130673567454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2048,12288,0.03758080005645752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2048,4096,0.015667200088500977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2048,3072,0.01437013347943624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2048,3584,0.01460906664530436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2048,2560,0.011980799833933513
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2048,6144,0.020923733711242676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2048,5120,0.01781760056813558
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2048,2048,0.011673600474993388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2048,1536,0.009523199995358785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2048,1024,0.007782400151093801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2048,512,0.008874666690826417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2048,768,0.00955733358860016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2048,128,0.0035157332817713416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2048,256,0.004232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2048,65536,0.17134933471679686
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2048,64,0.0032426667710145317
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,2048,32,0.0032768001159032187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1536,16384,0.037273601690928145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1536,12288,0.02935466567675273
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1536,8192,0.020855466524759926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1536,10240,0.02525866627693176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1536,7168,0.020002132654190062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1536,5120,0.014643200238545737
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1536,6144,0.01648640036582947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1536,4096,0.014028799533843995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1536,3584,0.013038933277130127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1536,3072,0.011195733149846395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1536,2560,0.010717866818110149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1536,2048,0.008601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1536,1536,0.009454933802286784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1536,65536,0.13271040121714275
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1536,1024,0.007714133461316426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1536,768,0.006894933183987935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1536,512,0.0055978665749231975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1536,256,0.004095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1536,128,0.0034133332471052804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1536,64,0.0031744000812371576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1536,32,0.00314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1024,12288,0.021333332856496176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1024,16384,0.02693120042483012
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1024,10240,0.018636800845464072
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1024,8192,0.016247466206550598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1024,7168,0.01460906664530436
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1024,5120,0.012390399972597759
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1024,6144,0.013346133629480996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1024,4096,0.011332266529401143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1024,65536,0.08744959831237793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1024,3584,0.010513066252072652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1024,3072,0.012185600399971009
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1024,2560,0.008191999793052674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1024,2048,0.00942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1024,1536,0.00795306662718455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1024,1024,0.007577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1024,768,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1024,512,0.0054613331953684485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1024,256,0.003993600110212962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1024,128,0.0034815999368826545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1024,64,0.0031061333914597826
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,1024,32,0.0030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,768,12288,0.019217065970102944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,768,16384,0.022732800245285033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,768,10240,0.017271467049916587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,768,8192,0.01546239952246348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,768,7168,0.014916266997655234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,768,65536,0.06843732992808024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,768,4096,0.011741866668065388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,768,5120,0.013380266229311624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,768,6144,0.014199466506640116
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,768,3072,0.009727999567985535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,768,3584,0.0105813334385554
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,768,2560,0.007884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,768,2048,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,768,1536,0.007577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,768,1024,0.006348800162474315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,768,768,0.006621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,768,512,0.005427200098832448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,768,256,0.0045738667249679565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,768,128,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,768,64,0.0030037333567937215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,768,32,0.0029696000119050344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,512,16384,0.015598932902018229
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,512,10240,0.012083199620246888
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,512,12288,0.013312000036239623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,512,65536,0.04997119903564453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,512,8192,0.010990933577219645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,512,7168,0.010308266679445902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,512,6144,0.009011200070381165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,512,5120,0.008191999793052674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,512,4096,0.007714133461316426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,512,3584,0.007338666419188182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,512,3072,0.008157866696516674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,512,2560,0.007884799937407176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,512,1536,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,512,1024,0.006007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,512,2048,0.007202133536338806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,512,512,0.005700266857941946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,512,768,0.006587733328342438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,512,256,0.004539733131726583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,512,64,0.0030037333567937215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,512,128,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,512,32,0.0029696000119050344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,256,12288,0.009250133236249288
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,256,10240,0.008396800359090168
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,256,65536,0.027067732810974122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,256,16384,0.011264000336329143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,256,7168,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,256,8192,0.0076117331782976794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,256,6144,0.007167999943097432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,256,5120,0.006894933183987935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,256,4096,0.006690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,256,3584,0.007236266632874806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,256,3072,0.0065194666385650635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,256,2048,0.00628053347269694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,256,2560,0.007748266557852428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,256,1536,0.005905066430568695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,256,1024,0.00580266664425532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,256,512,0.005529599885145823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,256,768,0.006621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,256,256,0.0044714664419492085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,256,128,0.0033791999022165934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,256,64,0.0029696000119050344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,128,16384,0.008362666765848795
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,128,65536,0.01672533353169759
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,128,12288,0.008430932958920796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,256,32,0.0029696000119050344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,128,10240,0.007406933108965556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,128,8192,0.007133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,128,7168,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,128,6144,0.006792533397674561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,128,5120,0.006963199873765309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,128,3584,0.006417066852251689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,128,4096,0.006621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,128,2560,0.007714133461316426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,128,3072,0.006007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,128,1536,0.005870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,128,2048,0.0065194666385650635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,128,1024,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,128,512,0.0054613331953684485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,128,768,0.0065194666385650635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,128,256,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,128,128,0.0033791999022165934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,128,64,0.0029696000119050344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,128,32,0.002935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,64,16384,0.008260266482830047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,64,65536,0.012970667084058127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,64,12288,0.007782400151093801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,64,10240,0.007441066702206929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,64,8192,0.00730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,64,7168,0.007338666419188182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,64,5120,0.007338666419188182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,64,4096,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,64,3584,0.006587733328342438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,64,3072,0.006382933259010315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,64,6144,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,64,2560,0.006109866499900818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,64,2048,0.006109866499900818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,64,1536,0.005836800237496694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,64,1024,0.005154133339722951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,64,768,0.005427200098832448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,64,256,0.004437333345413208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,64,512,0.005393066505591074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,64,64,0.00290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,64,32,0.00290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,64,128,0.0035157332817713416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,32,65536,0.01088853379090627
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,32,12288,0.007714133461316426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,32,16384,0.008089600006739299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,32,10240,0.007338666419188182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,32,8192,0.006587733328342438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,32,7168,0.007133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,32,6144,0.006690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,32,4096,0.00631466656923294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,32,3584,0.006382933259010315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,32,5120,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,32,3072,0.006075733403364817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,32,2560,0.006007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,32,2048,0.005905066430568695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,32,1536,0.005734399954477946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,32,1024,0.004915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,32,768,0.005393066505591074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,32,512,0.0052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,32,64,0.00290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,32,256,0.0046079998215039575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,32,128,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,8,32,32,0.002867199977238973
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,65536,4096,0.32447147369384766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,65536,6144,0.48834559122721355
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,65536,7168,0.566818110148112
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,65536,3584,0.29498027165730795
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,65536,8192,0.6507861455281575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,65536,2560,0.23350613911946616
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,65536,3072,0.2789717356363932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,65536,5120,0.41403732299804685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,65536,1024,0.10260480244954426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,65536,2048,0.19172693888346354
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,65536,768,0.07287466526031494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,65536,1536,0.1497429370880127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,65536,256,0.026794666051864625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,65536,128,0.014677332838376364
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,65536,64,0.008840533097585042
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,65536,32,0.013516799608866373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,65536,512,0.052121599515279136
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,16384,12288,0.24275627136230468
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,16384,10240,0.2061994711558024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,16384,16384,0.31836160024007165
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,16384,8192,0.17100799878438314
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,65536,10240,0.7973887761433919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,16384,7168,0.1497770627339681
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,16384,6144,0.131549866994222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,16384,5120,0.12257280349731445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,16384,2048,0.05495466788609823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,16384,2560,0.06877866586049398
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,16384,3072,0.07871146996816
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,65536,12288,0.9469952265421548
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,16384,4096,0.09902079900105795
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,16384,3584,0.0904533306757609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,16384,1024,0.031163734197616578
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,16384,768,0.02307413419087728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,16384,512,0.016315733393033348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,16384,1536,0.0400383989016215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,16384,256,0.00993280013402303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,16384,128,0.005085866649945577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,16384,64,0.004130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,16384,32,0.0037546666959921518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,12288,7168,0.11649706363677978
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,12288,8192,0.13018453121185303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,12288,16384,0.24210774103800453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,12288,10240,0.1577642599741618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,65536,16384,1.2667221069335937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,12288,6144,0.09983999729156494
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,12288,12288,0.1887231985727946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,12288,5120,0.08567466735839843
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,12288,4096,0.07031466960906982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,12288,1536,0.03126613299051921
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,12288,2560,0.04696746667226155
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,12288,1024,0.02249386707941691
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,16384,65536,1.2162731170654297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,12288,2048,0.03863893349965413
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,12288,3584,0.0575488011042277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,12288,3072,0.05519359906514486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,12288,768,0.01996799906094869
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,12288,512,0.01443839967250824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,12288,256,0.007782400151093801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,12288,128,0.004642133414745331
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,12288,64,0.0038912000755469004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,12288,32,0.00631466656923294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,10240,10240,0.1316864013671875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,10240,12288,0.15506772994995116
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,10240,8192,0.11011412938435872
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,10240,16384,0.20183040301005045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,10240,3584,0.04969813426335652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,10240,5120,0.08147626717885335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,10240,4096,0.06082559823989868
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,10240,6144,0.0886783997217814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,10240,7168,0.09751893679300944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,10240,2560,0.03775146802266439
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,10240,3072,0.04795733292897542
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,10240,2048,0.03413333495457967
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,10240,1536,0.027374933163324993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,10240,512,0.012936533490816752
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,10240,256,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,10240,1024,0.020070399840672812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,10240,128,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,10240,768,0.018875734011332194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,10240,64,0.0037546666959921518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,10240,32,0.006621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,8192,8192,0.09216000239054362
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,8192,7168,0.08219306468963623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,8192,12288,0.12666880289713542
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,8192,5120,0.0589141329129537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,8192,6144,0.06925653616587321
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,8192,4096,0.051268267631530764
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,8192,10240,0.10861226717631023
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,8192,3584,0.04645546674728394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,8192,2560,0.03160746693611145
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,8192,3072,0.040584532419840495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,8192,1536,0.02218666672706604
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,8192,16384,0.1658880074818929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,8192,1024,0.016657066345214844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,8192,512,0.013004799683888754
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,8192,768,0.01751040021578471
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,8192,2048,0.029149866104125975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,8192,128,0.004300799965858459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,12288,65536,0.9135445276896158
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,8192,64,0.0035157332817713416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,8192,256,0.00576853354771932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,8192,32,0.00628053347269694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,7168,10240,0.10134186744689941
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,7168,7168,0.07563947041829427
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,7168,16384,0.15069866180419922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,7168,6144,0.06369280020395915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,8192,65536,0.6180181503295898
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,7168,8192,0.08116906483968099
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,7168,4096,0.04652373393376668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,7168,12288,0.13056000073750812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,7168,5120,0.05263359944025675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,10240,65536,0.7597738901774089
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,7168,3584,0.04150613149007161
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,7168,3072,0.03628373146057129
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,7168,2560,0.03092479904492696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,7168,2048,0.025565866629282636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,7168,1024,0.014984533190727234
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,7168,1536,0.01867093245188395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,7168,512,0.012151466806729634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,7168,256,0.005256533126036326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,7168,768,0.016827734311421712
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,7168,64,0.006792533397674561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,7168,32,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,7168,128,0.004095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,6144,8192,0.06816426912943521
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,6144,12288,0.09717760086059571
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,6144,16384,0.12588373025258381
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,6144,7168,0.057275732358296715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,6144,3584,0.03375786542892456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,6144,5120,0.04235946734746297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,6144,6144,0.05021013418833414
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,6144,4096,0.03536213239034017
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,6144,10240,0.08055466810862223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,6144,3072,0.029491200049718218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,6144,2560,0.024951465924580894
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,6144,2048,0.020855466524759926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,6144,1024,0.01160533328851064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,6144,1536,0.016657066345214844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,6144,512,0.007338666419188182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,6144,256,0.0055978665749231975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,6144,768,0.00942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,6144,64,0.006792533397674561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,6144,128,0.003925333420435587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,7168,65536,0.5495466868082682
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,6144,32,0.00628053347269694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,5120,8192,0.057378133138020836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,5120,10240,0.0679253339767456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,5120,16384,0.10693973700205486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,5120,12288,0.08028159936269125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,5120,3584,0.028603732585906982
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,5120,4096,0.03362133502960205
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,5120,5120,0.035396265983581546
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,5120,6144,0.04188160101572673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,5120,7168,0.05358933210372925
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,5120,2560,0.02167466680208842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,5120,2048,0.017271467049916587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,5120,3072,0.02307413419087728
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,5120,1536,0.014882133404413859
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,5120,1024,0.011127466956774395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,5120,768,0.01372160017490387
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,6144,65536,0.46551040013631184
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,5120,512,0.006109866499900818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,5120,256,0.0054613331953684485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,5120,64,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,5120,128,0.004369066655635833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,5120,32,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,4096,12288,0.06458026568094889
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,4096,10240,0.05451093514760336
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,4096,8192,0.04659200112024943
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,5120,65536,0.3889493306477865
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,4096,7168,0.040174933274586995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,4096,5120,0.03300693432490031
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,4096,6144,0.034884266058603924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,4096,4096,0.026282666126887004
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,4096,3584,0.02471253275871277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,4096,16384,0.09741653601328532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,4096,3072,0.021572266022364298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,4096,2048,0.015735466281572977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,4096,1024,0.010308266679445902
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,4096,2560,0.018739199638366698
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,4096,768,0.013004799683888754
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,4096,1536,0.013004799683888754
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,4096,256,0.0052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,4096,512,0.005905066430568695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,4096,128,0.004300799965858459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,4096,64,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,4096,32,0.0032768001159032187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3584,8192,0.04358826478322347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3584,12288,0.05980159838994344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3584,7168,0.03857066631317139
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3584,10240,0.05113173325856527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3584,16384,0.07833600044250488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3584,5120,0.02969599962234497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,4096,65536,0.3122858683268229
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3584,6144,0.0343722661336263
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3584,4096,0.026009599367777508
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3584,3584,0.02430293361345927
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3584,2048,0.014643200238545737
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3584,3072,0.020275199413299562
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3584,2560,0.019012266397476198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3584,1536,0.012151466806729634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3584,1024,0.00955733358860016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3584,768,0.01181013286113739
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3584,256,0.0052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3584,512,0.006109866499900818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3584,128,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3584,64,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3584,65536,0.279586124420166
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3584,32,0.0032426667710145317
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3072,12288,0.05277013381322225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3072,10240,0.04727466503779094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3072,6144,0.03269973397254944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3072,16384,0.06621866623560588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3072,7168,0.0340992013613383
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3072,8192,0.03819520076115926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3072,5120,0.02590720057487488
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3072,4096,0.02481493353843689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3072,3584,0.02187946637471517
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3072,3072,0.01812480092048645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3072,2560,0.016657066345214844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3072,2048,0.013755733768145243
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3072,1536,0.01153706709543864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3072,768,0.010683733224868774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3072,512,0.005870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3072,1024,0.008055466910203297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3072,256,0.0045056000351905824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3072,128,0.0036864000062147772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3072,64,0.0032768001159032187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3072,32,0.0032768001159032187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,3072,65536,0.24674986203511556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2560,16384,0.057719465096791586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2560,10240,0.03925333420435588
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2560,8192,0.03351893424987793
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2560,12288,0.045670398076375324
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2560,7168,0.030003199974695843
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2560,6144,0.026726400852203368
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2560,4096,0.02106026609738668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2560,5120,0.02484906713167826
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2560,3584,0.01945599913597107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2560,3072,0.017407999436060587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2560,2560,0.015974400440851848
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2560,2048,0.014097066720326743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2560,1536,0.010137599706649781
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2560,1024,0.007782400151093801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2560,65536,0.20428800582885742
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2560,768,0.010513066252072652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2560,512,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2560,256,0.004334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2560,128,0.003583999971548716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2560,64,0.0032426667710145317
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2560,32,0.0032085334261258446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2048,10240,0.030822400252024335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2048,16384,0.053179732958475744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2048,8192,0.027204267183939618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2048,7168,0.02757973273595174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2048,12288,0.037102933724721274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2048,6144,0.02054826617240906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2048,4096,0.01693013310432434
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2048,5120,0.017885865767796834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2048,3584,0.015633066495259605
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2048,3072,0.013687466581662497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2048,2560,0.011912533640861511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2048,2048,0.010820266604423524
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2048,1536,0.009591466188430786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2048,65536,0.17121280034383138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2048,1024,0.007748266557852428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2048,768,0.009523199995358785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2048,512,0.0056320001681645715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2048,256,0.004198400179545084
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2048,128,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2048,64,0.00314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1536,10240,0.024985599517822265
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,2048,32,0.0032085334261258446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1536,16384,0.037205334504445395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1536,8192,0.02109439969062805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1536,12288,0.02874026695887248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1536,5120,0.014847999811172486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1536,7168,0.01812480092048645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1536,6144,0.016520532965660095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1536,4096,0.013038933277130127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1536,3584,0.013038933277130127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1536,3072,0.011195733149846395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1536,2560,0.011400533715883891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1536,65536,0.13281280199686687
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1536,2048,0.008567466338475545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1536,1536,0.009454933802286784
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1536,1024,0.006621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1536,768,0.006963199873765309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1536,512,0.005836800237496694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1536,64,0.00314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1536,32,0.0032085334261258446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1536,256,0.004266666869322458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1536,128,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1024,10240,0.018397865692774455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1024,7168,0.014643200238545737
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1024,12288,0.02106026609738668
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1024,16384,0.026521599292755126
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1024,8192,0.015974400440851848
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1024,6144,0.013414399822553
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1024,5120,0.012288000186284382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1024,65536,0.08649386564890543
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1024,4096,0.010854400197664897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1024,3584,0.010752000411351522
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1024,2048,0.00942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1024,3072,0.011912533640861511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1024,2560,0.008226133386294047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1024,1024,0.007577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1024,1536,0.007748266557852428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1024,768,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1024,256,0.0046079998215039575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1024,512,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1024,128,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1024,64,0.0030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,1024,32,0.0030720000465710956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,768,16384,0.02263039946556091
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,768,10240,0.017544533809026083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,768,7168,0.014779733618100485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,768,8192,0.015359999736150107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,768,12288,0.01904639999071757
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,768,6144,0.014165332913398743
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,768,5120,0.013346133629480996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,768,65536,0.06758399804433188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,768,3584,0.010513066252072652
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,768,3072,0.009659733374913533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,768,4096,0.010854400197664897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,768,2560,0.007850666840871174
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,768,2048,0.006929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,768,1536,0.00727040022611618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,768,1024,0.006212266782919565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,768,768,0.006621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,768,512,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,768,256,0.0045056000351905824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,768,64,0.0029696000119050344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,768,128,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,768,32,0.0030037333567937215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,512,16384,0.01539413332939148
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,512,10240,0.012049067020416259
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,512,12288,0.013277866442998252
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,512,8192,0.010956799983978272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,512,7168,0.010137599706649781
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,512,65536,0.04908373355865479
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,512,5120,0.008021333316961924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,512,6144,0.008874666690826417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,512,3584,0.007167999943097432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,512,4096,0.007577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,512,2560,0.007782400151093801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,512,3072,0.007987200220425924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,512,2048,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,512,1536,0.006451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,512,1024,0.005973333120346069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,512,768,0.006587733328342438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,512,512,0.0055637334783871974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,512,256,0.0045056000351905824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,512,128,0.0034133332471052804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,512,64,0.0029696000119050344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,512,32,0.0030037333567937215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,256,65536,0.026385066906611125
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,256,16384,0.011434666315714518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,256,10240,0.008294400076071422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,256,12288,0.009181867043177288
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,256,7168,0.0073728000124295544
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,256,8192,0.0075434664885203045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,256,6144,0.007133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,256,5120,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,256,4096,0.0065536002318064375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,256,3072,0.006417066852251689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,256,3584,0.007236266632874806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,256,2048,0.006178133189678192
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,256,1536,0.00580266664425532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,256,2560,0.0076799998680750535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,256,1024,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,256,768,0.006621866424878438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,256,256,0.0044714664419492085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,256,512,0.005393066505591074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,256,128,0.0035157332817713416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,256,64,0.002935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,256,32,0.002935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,128,16384,0.008089600006739299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,128,65536,0.016452266772588094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,128,10240,0.007167999943097432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,128,12288,0.00747519979874293
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,128,8192,0.006792533397674561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,128,7168,0.006929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,128,6144,0.0065536002318064375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,128,5120,0.006724266707897187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,128,4096,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,128,3072,0.006212266782919565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,128,3584,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,128,2560,0.007645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,128,2048,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,128,1536,0.00576853354771932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,128,1024,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,128,768,0.0065536002318064375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,128,512,0.005358933409055074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,128,256,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,128,128,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,128,32,0.00290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,128,64,0.002935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,64,65536,0.012868266304334006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,64,16384,0.008191999793052674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,64,12288,0.007577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,64,10240,0.007406933108965556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,64,8192,0.007133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,64,6144,0.006690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,64,4096,0.006417066852251689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,64,5120,0.006963199873765309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,64,3584,0.006485333542029063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,64,7168,0.007167999943097432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,64,3072,0.006075733403364817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,64,2560,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,64,2048,0.006007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,64,1536,0.005700266857941946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,64,512,0.0053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,64,1024,0.0052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,64,768,0.0054613331953684485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,64,256,0.004369066655635833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,64,128,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,64,32,0.002935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,64,64,0.00290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,32,16384,0.008089600006739299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,32,12288,0.007577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,32,65536,0.0106495996316274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,32,10240,0.007236266632874806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,32,7168,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,32,8192,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,32,6144,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,32,5120,0.006724266707897187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,32,4096,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,32,3584,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,32,3072,0.005905066430568695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,32,2560,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,32,2048,0.005870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,32,1024,0.004846933484077454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,32,1536,0.005700266857941946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,32,768,0.005256533126036326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,32,512,0.005256533126036326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,32,128,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,32,256,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,32,64,0.00290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,4,32,32,0.0027989332874615988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,65536,5120,0.41117013295491533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,65536,4096,0.3203754742940267
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,65536,8192,0.6438229242960612
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,65536,3584,0.2937856038411458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,65536,6144,0.4834986686706543
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,65536,12288,0.9441279729207357
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,65536,2048,0.1933311939239502
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,65536,2560,0.23336960474650065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,65536,7168,0.5626197179158529
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,65536,768,0.07273813088734946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,65536,1024,0.10274133682250977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,65536,16384,1.2634112040201821
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,65536,512,0.048981332778930665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,65536,256,0.027955200274785357
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,65536,128,0.022937599817911783
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,65536,64,0.010547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,65536,32,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,65536,3072,0.25094827016194665
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,65536,1536,0.14854826927185058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,65536,10240,0.7927808125813802
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,16384,10240,0.20469759305318197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,16384,8192,0.16926719347635905
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,16384,12288,0.24023040135701498
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,16384,6144,0.1299455960591634
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,16384,7168,0.16247466405232747
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,16384,4096,0.0897706667582194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,16384,3584,0.08209066390991211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,16384,3072,0.07830186684926352
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,16384,1024,0.029047467311223346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,16384,5120,0.12233386834462483
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,16384,1536,0.04235946734746297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,16384,768,0.023142399390538533
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,16384,512,0.01525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,16384,2560,0.06925653616587321
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,16384,2048,0.054749866326649986
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,16384,128,0.0055637334783871974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,16384,64,0.004300799965858459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,16384,16384,0.3164501190185547
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,16384,256,0.009352533022562663
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,16384,32,0.006417066852251689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,12288,6144,0.10393599669138591
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,12288,10240,0.15670612653096516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,12288,12288,0.1829546610514323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,12288,5120,0.08495786984761557
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,12288,7168,0.11356159845987956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,12288,4096,0.07041707038879394
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,12288,3584,0.06263466676076254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,12288,8192,0.14493014017740885
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,12288,3072,0.05509119828542074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,12288,1536,0.031197865804036457
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,12288,2048,0.03887786865234375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,12288,768,0.020070399840672812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,12288,512,0.013516799608866373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,12288,1024,0.022528000672658286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,12288,256,0.008089600006739299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,12288,2560,0.04300800164540609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,12288,16384,0.2401279926300049
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,12288,128,0.0055637334783871974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,12288,64,0.005836800237496694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,12288,32,0.0054613331953684485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,10240,10240,0.13056000073750812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,10240,8192,0.10905600388844808
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,10240,6144,0.08850773175557455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,10240,5120,0.07659520308176676
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,10240,12288,0.16960852940877277
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,16384,65536,1.222109858194987
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,10240,7168,0.09704106648763021
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,10240,4096,0.06103039979934692
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,10240,16384,0.20118187268575033
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,10240,2048,0.03372373183568318
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,10240,1536,0.027101866404215497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,10240,3584,0.054545064767201744
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,10240,2560,0.04082346757253011
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,10240,3072,0.04782079855600993
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,10240,1024,0.01996799906094869
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,10240,512,0.012868266304334006
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,10240,128,0.005051733553409576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,10240,256,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,10240,768,0.018568533658981323
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,10240,64,0.0053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,10240,32,0.004915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,8192,12288,0.12619093259175618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,8192,16384,0.16803840001424153
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,8192,8192,0.08929279645284018
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,8192,7168,0.07946240107218425
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,8192,10240,0.10816853046417237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,10240,65536,0.7577258427937825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,8192,6144,0.07662933667500814
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,8192,2560,0.034747731685638425
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,8192,5120,0.05809493462244669
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,12288,65536,0.9212245305379232
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,8192,3584,0.046216531594594316
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,8192,3072,0.04078933397928874
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,8192,4096,0.05130240122477213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,8192,2048,0.028842665751775104
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,8192,1024,0.015837867061297098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,8192,768,0.01751040021578471
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,8192,1536,0.020821332931518555
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,8192,512,0.01269760032494863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,8192,256,0.005393066505591074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,8192,128,0.004881066580613455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,8192,32,0.0045056000351905824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,8192,64,0.00481279989083608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,7168,6144,0.06318080027898153
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,7168,7168,0.07106560071309408
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,7168,16384,0.14752426147460937
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,8192,65536,0.6189738591512044
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,7168,8192,0.08120319843292237
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,7168,10240,0.10178559621175129
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,7168,3072,0.0361130674680074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,7168,12288,0.11813546816507976
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,7168,4096,0.043144532044728595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,7168,5120,0.051950931549072266
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,7168,3584,0.037614933649698895
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,7168,2048,0.026077866554260254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,7168,2560,0.030719999472300214
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,7168,1536,0.018773333231608073
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,7168,768,0.016827734311421712
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,7168,1024,0.01495039959748586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,7168,256,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,7168,512,0.012185600399971009
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,7168,32,0.004369066655635833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,7168,128,0.005017599960168203
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,7168,64,0.0046079998215039575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,6144,12288,0.09700693289438883
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,6144,8192,0.06799360116322836
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,6144,10240,0.08038400014241537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,6144,4096,0.03822933435440064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,6144,5120,0.04713813463846843
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,6144,3584,0.033928533395131424
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,6144,6144,0.04980053504308064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,6144,7168,0.05751466751098633
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,6144,16384,0.12595199743906657
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,6144,3072,0.02648746569951375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,6144,2560,0.024644267559051514
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,6144,2048,0.02068480054537455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,6144,512,0.007202133536338806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,6144,1536,0.016657066345214844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,6144,768,0.010171733299891154
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,6144,1024,0.012663466731707254
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,6144,256,0.0054954667886098225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,6144,64,0.004334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,6144,128,0.004744533201058706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,7168,65536,0.5490687688191731
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,6144,32,0.0039594667653242745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,5120,12288,0.08038400014241537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,5120,10240,0.0679253339767456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,6144,65536,0.46373545328776045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,5120,8192,0.057719465096791586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,5120,4096,0.03310933311780294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,5120,3584,0.028637866179148357
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,5120,5120,0.039082666238149
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,5120,16384,0.10680320262908935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,5120,3072,0.02505386670430501
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,5120,6144,0.04157439867655437
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,5120,7168,0.053623465696970615
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,5120,2560,0.020002132654190062
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,5120,1536,0.014711466431617738
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,5120,2048,0.01641813317934672
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,5120,1024,0.011366400122642516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,5120,768,0.014472533265749613
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,5120,512,0.006109866499900818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,5120,128,0.004232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,5120,256,0.0053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,5120,32,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,5120,64,0.0034815999368826545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,4096,10240,0.054647465546925865
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,4096,6144,0.034918399651845296
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,4096,7168,0.04171093304951985
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,4096,16384,0.08820052941640219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,4096,5120,0.03269973397254944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,4096,8192,0.04706986745198567
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,4096,4096,0.024064000447591147
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,4096,3584,0.024439465999603272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,4096,12288,0.0681984027226766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,4096,2560,0.017237333456675212
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,4096,3072,0.021708800395329794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,4096,1536,0.0130730668703715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,4096,1024,0.01027413308620453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,4096,2048,0.01443839967250824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,5120,65536,0.3875157356262207
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,4096,768,0.01269760032494863
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,4096,512,0.00580266664425532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,4096,128,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,4096,64,0.0032768001159032187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,4096,256,0.0053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,4096,32,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3584,12288,0.06256639957427979
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3584,8192,0.04474879900614421
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3584,7168,0.03867306709289551
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3584,10240,0.053282133738199865
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3584,5120,0.031675734122594196
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3584,16384,0.08253440062204996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3584,4096,0.025361067056655882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3584,3072,0.020855466524759926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3584,6144,0.033655468622843424
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3584,3584,0.02362026572227478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,4096,65536,0.30996481577555335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3584,2560,0.01648640036582947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3584,1536,0.012185600399971009
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3584,1024,0.008942932883898417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3584,2048,0.01525759994983673
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3584,512,0.005700266857941946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3584,768,0.011707733074824016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3584,128,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3584,256,0.004983466863632202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3584,64,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3584,32,0.0034815999368826545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3072,10240,0.04440746704737346
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3072,6144,0.028398933013280232
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3072,7168,0.03211946686108907
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3584,65536,0.2795178731282552
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3072,8192,0.037444265683492024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3072,12288,0.05232640107472738
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3072,16384,0.07028053601582845
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3072,5120,0.026658133665720625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3072,3584,0.020206934213638304
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3072,4096,0.02068480054537455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3072,1024,0.008669867118199667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3072,3072,0.01843199928601583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3072,2560,0.01621333360671997
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3072,2048,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3072,1536,0.011434666315714518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3072,768,0.010615467031796774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3072,512,0.005666133264700572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3072,256,0.0046079998215039575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3072,32,0.0032085334261258446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3072,64,0.0033791999022165934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3072,128,0.003618133316437403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2560,12288,0.043246932824452716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2560,16384,0.059699201583862306
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2560,10240,0.037102933724721274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,3072,65536,0.2469546635945638
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2560,7168,0.028194133440653486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2560,6144,0.02481493353843689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2560,8192,0.031061333417892457
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2560,3584,0.018158932526906334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2560,4096,0.019729065895080566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2560,5120,0.02362026572227478
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2560,3072,0.016657066345214844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2560,2560,0.013550933202107748
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2560,2048,0.012288000186284382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2560,1024,0.007714133461316426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2560,1536,0.010752000411351522
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2560,768,0.010547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2560,512,0.005666133264700572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2560,256,0.0044714664419492085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2560,65536,0.2032639980316162
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2560,32,0.0032426667710145317
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2560,128,0.0035157332817713416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2560,64,0.0032085334261258446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2048,16384,0.04829866488774617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2048,12288,0.036761601765950516
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2048,7168,0.02290346622467041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2048,8192,0.029252266883850096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2048,10240,0.03092479904492696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2048,6144,0.020514132579167683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2048,5120,0.017988266547520955
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2048,4096,0.016759467124938966
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2048,3584,0.01546239952246348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2048,3072,0.013209600249926248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2048,2560,0.011878400047620138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2048,2048,0.011639466881752015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2048,768,0.00955733358860016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2048,1536,0.009591466188430786
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2048,1024,0.0076117331782976794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2048,512,0.0056320001681645715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2048,256,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2048,65536,0.17278292973836262
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2048,128,0.0034133332471052804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2048,32,0.0032085334261258446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,2048,64,0.0032085334261258446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1536,10240,0.024541866779327393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1536,12288,0.028501333793004353
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1536,16384,0.0369322657585144
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1536,6144,0.016452266772588094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1536,8192,0.02290346622467041
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1536,4096,0.012970667084058127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1536,7168,0.01812480092048645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1536,3584,0.013038933277130127
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1536,5120,0.014711466431617738
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1536,2560,0.010547199845314026
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1536,1536,0.009489066402117411
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1536,3072,0.011912533640861511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1536,2048,0.008499200145403545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1536,65536,0.13216426372528076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1536,768,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1536,512,0.0056320001681645715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1536,1024,0.006485333542029063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1536,128,0.0034474665919939675
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1536,256,0.004232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1536,64,0.0030720000465710956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1536,32,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1024,16384,0.02635093331336975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1024,12288,0.020957867304484047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1024,8192,0.016383999586105348
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1024,7168,0.014506666858990987
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1024,10240,0.018295466899871826
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1024,6144,0.013346133629480996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1024,5120,0.012219732999801636
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1024,4096,0.011434666315714518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1024,65536,0.08697173595428467
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1024,3072,0.01181013286113739
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1024,3584,0.0106495996316274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1024,2560,0.008055466910203297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1024,1024,0.0075434664885203045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1024,1536,0.0076117331782976794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1024,768,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1024,256,0.0045738667249679565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1024,512,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1024,2048,0.009284266829490661
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1024,64,0.0030037333567937215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1024,128,0.003618133316437403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,768,16384,0.02222080032030741
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,1024,32,0.0030037333567937215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,768,12288,0.019012266397476198
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,768,7168,0.014779733618100485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,768,6144,0.014028799533843995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,768,10240,0.017237333456675212
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,768,8192,0.015359999736150107
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,768,65536,0.06676479975382486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,768,3584,0.010752000411351522
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,768,4096,0.010922666390736897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,768,5120,0.013209600249926248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,768,3072,0.00955733358860016
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,768,2560,0.007816533247629803
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,768,2048,0.006894933183987935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,768,768,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,768,1024,0.0075434664885203045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,768,512,0.005905066430568695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,768,256,0.0045056000351905824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,768,1536,0.007202133536338806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,768,128,0.0034133332471052804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,768,64,0.0029696000119050344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,768,32,0.0030037333567937215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,512,16384,0.015564800302187601
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,512,12288,0.013175466656684875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,512,10240,0.011980799833933513
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,512,7168,0.010137599706649781
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,512,65536,0.04829866488774617
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,512,5120,0.007918933530648549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,512,6144,0.008669867118199667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,512,8192,0.010922666390736897
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,512,2560,0.007748266557852428
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,512,3584,0.007133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,512,4096,0.007577600081761678
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,512,3072,0.007918933530648549
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,512,2048,0.006963199873765309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,512,1536,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,512,1024,0.0075434664885203045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,512,512,0.0055637334783871974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,512,768,0.006587733328342438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,512,256,0.0045056000351905824
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,512,128,0.0034133332471052804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,512,64,0.0029696000119050344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,512,32,0.0029696000119050344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,256,16384,0.011025066177050274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,256,65536,0.02604373296101888
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,256,12288,0.009045333663622538
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,256,8192,0.007406933108965556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,256,7168,0.007236266632874806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,256,6144,0.006963199873765309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,256,5120,0.006758399804433187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,256,10240,0.008191999793052674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,256,4096,0.006485333542029063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,256,3584,0.006758399804433187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,256,3072,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,256,2560,0.007645866771539052
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,256,2048,0.006041599810123444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,256,1024,0.0075434664885203045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,256,1536,0.005734399954477946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,256,512,0.005427200098832448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,256,256,0.004437333345413208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,256,768,0.006587733328342438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,256,128,0.0033791999022165934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,256,64,0.0030037333567937215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,256,32,0.0029696000119050344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,128,65536,0.01628159979979197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,128,16384,0.008089600006739299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,128,12288,0.007441066702206929
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,128,10240,0.007133866846561432
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,128,7168,0.007099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,128,8192,0.006792533397674561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,128,4096,0.006382933259010315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,128,6144,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,128,5120,0.006587733328342438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,128,3584,0.006826666494210561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,128,3072,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,128,2560,0.0075434664885203045
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,128,2048,0.006144000093142191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,128,1024,0.0075093333919843035
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,128,768,0.006724266707897187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,128,512,0.0053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,128,1536,0.0056320001681645715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,128,256,0.004369066655635833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,128,128,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,128,64,0.002935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,128,32,0.00290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,64,65536,0.012731732924779257
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,64,16384,0.008226133386294047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,64,12288,0.007714133461316426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,64,8192,0.007099733253320058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,64,10240,0.007406933108965556
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,64,7168,0.007065600156784058
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,64,5120,0.006963199873765309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,64,6144,0.0065536002318064375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,64,4096,0.006382933259010315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,64,3584,0.006417066852251689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,64,3072,0.006212266782919565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,64,2560,0.006212266782919565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,64,1536,0.0055637334783871974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,64,1024,0.004983466863632202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,64,768,0.005427200098832448
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,64,2048,0.006007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,64,512,0.005358933409055074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,64,256,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,64,128,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,64,64,0.00290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,64,32,0.00290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,32,65536,0.011127466956774395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,32,16384,0.008157866696516674
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,32,10240,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,32,8192,0.006690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,32,7168,0.006826666494210561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,32,12288,0.007202133536338806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,32,6144,0.006348800162474315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,32,4096,0.006485333542029063
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,32,5120,0.0065194666385650635
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,32,3584,0.00631466656923294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,32,3072,0.00580266664425532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,32,2560,0.00580266664425532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,32,2048,0.0056320001681645715
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,32,1536,0.0054613331953684485
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,32,768,0.005256533126036326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,32,1024,0.00481279989083608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,32,512,0.005085866649945577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,32,128,0.0033791999022165934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,32,256,0.004300799965858459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,32,32,0.002935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,2,32,64,0.0030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,65536,7168,0.56627197265625
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,65536,4096,0.30658559799194335
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,65536,10240,0.7971498489379882
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,65536,5120,0.4037290573120117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,65536,3584,0.30122667948404946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,65536,6144,0.48390827178955076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,65536,1536,0.13527040481567382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,65536,2560,0.21845332781473795
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,65536,12288,0.9466880162556967
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,65536,768,0.0731818675994873
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,65536,8192,0.6448810577392579
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,65536,1024,0.09437867005666098
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,65536,512,0.05734399954477946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,65536,3072,0.23528107007344565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,65536,2048,0.17527467409769695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,65536,256,0.02754559914271037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,65536,128,0.013380266229311624
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,65536,64,0.014335999886194864
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,65536,32,0.007782400151093801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,65536,16384,1.2526933034261067
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,16384,8192,0.1726122697194417
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,16384,5120,0.1218559980392456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,16384,10240,0.20319573084513345
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,16384,4096,0.09847466945648194
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,16384,6144,0.13359786669413248
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,16384,7168,0.1467733383178711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,16384,3584,0.07809706528981528
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,16384,2560,0.066594131787618
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,16384,3072,0.07847253481547037
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,16384,12288,0.2401279926300049
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,16384,1024,0.02815999984741211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,16384,1536,0.04334933360417684
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,16384,768,0.02239146629969279
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,16384,2048,0.054306133588155114
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,16384,16384,0.32068265279134117
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,16384,256,0.00993280013402303
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,16384,512,0.016588800152142844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,16384,32,0.004334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,16384,128,0.005051733553409576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,16384,64,0.007782400151093801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,12288,8192,0.12933119932810466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,12288,7168,0.11400533517201741
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,12288,10240,0.15540906588236492
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,12288,5120,0.0839680035909017
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,12288,4096,0.06666239897410074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,12288,6144,0.09963520367940268
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,12288,3584,0.06301013231277466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,12288,12288,0.186845858891805
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,12288,3072,0.05492053429285685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,12288,2560,0.042530135313669844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,12288,1536,0.029627732435862225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,12288,16384,0.24326826731363932
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,12288,768,0.01689599951108297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,12288,2048,0.03857066631317139
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,12288,1024,0.021401600042978922
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,12288,256,0.010069333513577779
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,12288,512,0.011434666315714518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,12288,64,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,12288,128,0.004539733131726583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,12288,32,0.003925333420435587
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,10240,10240,0.12943360010782878
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,10240,12288,0.15339520772298176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,16384,65536,1.254638926188151
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,10240,16384,0.20763306617736815
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,10240,8192,0.10714453061421711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,10240,6144,0.07973546981811523
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,10240,7168,0.09216000239054362
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,10240,4096,0.06085973183314005
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,10240,5120,0.06683306694030762
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,10240,3584,0.054271999994913736
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,10240,2560,0.040516265233357746
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,10240,1024,0.017646932601928712
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,10240,1536,0.022835199038187662
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,10240,768,0.014643200238545737
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,10240,512,0.011741866668065388
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,10240,3072,0.04778666496276855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,10240,128,0.004300799965858459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,10240,2048,0.0318122665087382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,10240,256,0.00628053347269694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,10240,64,0.004539733131726583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,10240,32,0.0034815999368826545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,8192,7168,0.07652693589528402
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,8192,10240,0.10571093559265136
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,8192,8192,0.0879957358042399
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,8192,16384,0.16670719782511395
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,8192,6144,0.06676479975382486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,8192,4096,0.05106346607208252
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,8192,5120,0.0566271980603536
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,8192,3584,0.04198400179545085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,8192,12288,0.13564586639404297
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,8192,3072,0.040447998046875
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,8192,2560,0.034065067768096924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,8192,1536,0.01935360034306844
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,8192,768,0.012834133704503379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,8192,2048,0.024439465999603272
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,8192,1024,0.015735466281572977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,12288,65536,0.9127936045328775
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,8192,512,0.0105813334385554
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,8192,256,0.0055637334783871974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,8192,128,0.004983466863632202
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,8192,64,0.004232533276081085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,8192,32,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,7168,10240,0.09519786834716797
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,7168,12288,0.11110400358835856
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,7168,8192,0.08021333217620849
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,7168,16384,0.14895787239074706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,7168,6144,0.05925546487172445
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,7168,5120,0.05079040129979452
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,7168,3584,0.04061866601308187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,7168,4096,0.046011734008789065
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,7168,2560,0.02836479942003886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,7168,7168,0.06806186834971109
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,7168,3072,0.03566933472951253
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,10240,65536,0.756121571858724
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,7168,2048,0.02484906713167826
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,7168,512,0.009727999567985535
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,7168,1024,0.013516799608866373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,7168,1536,0.019285333156585694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,7168,128,0.004539733131726583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,7168,768,0.012014933427174886
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,7168,64,0.0052906667192777
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,7168,256,0.0053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,7168,32,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,6144,7168,0.05700266758600871
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,6144,10240,0.08034986654917399
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,8192,65536,0.6294186909993489
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,6144,16384,0.12506453196207684
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,6144,8192,0.06758399804433188
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,6144,5120,0.04689919948577881
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,6144,12288,0.09400320053100586
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,6144,6144,0.05007359981536865
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,6144,3584,0.03375786542892456
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,6144,2560,0.024541866779327393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,6144,4096,0.03860479990641276
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,6144,1536,0.01672533353169759
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,6144,3072,0.02648746569951375
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,6144,2048,0.02146986722946167
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,6144,1024,0.012356266379356384
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,6144,768,0.010205866893132527
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,6144,512,0.006656000018119812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,7168,65536,0.5636437098185222
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,6144,256,0.0046079998215039575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,6144,128,0.0040618665516376495
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,6144,64,0.003822933385769526
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,6144,32,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,5120,6144,0.04157439867655437
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,5120,10240,0.0676522652308146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,6144,65536,0.4638037363688151
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,5120,7168,0.048401065667470294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,5120,8192,0.05765120188395182
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,5120,12288,0.08919040362040201
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,5120,16384,0.10697387059529621
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,5120,5120,0.03515733480453491
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,5120,4096,0.03307519952456157
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,5120,3584,0.028569600979487103
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,5120,3072,0.025088000297546386
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,5120,2560,0.02167466680208842
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,5120,1024,0.01129813293615977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,5120,2048,0.01648640036582947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,5120,1536,0.013516799608866373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,5120,768,0.00942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,5120,256,0.004266666869322458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,5120,128,0.00365226666132609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,5120,512,0.006007466713587443
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,5120,64,0.0035157332817713416
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,5120,32,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,5120,65536,0.39601494471232096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,4096,16384,0.0881322701772054
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,4096,7168,0.03993599812189738
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,4096,8192,0.048640000820159915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,4096,10240,0.05522773265838623
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,4096,12288,0.06485333442687988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,4096,5120,0.032767999172210696
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,4096,6144,0.034747731685638425
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,4096,4096,0.02621440092722575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,4096,3584,0.024541866779327393
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,4096,3072,0.01952426632245382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,4096,1536,0.011912533640861511
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,4096,512,0.005905066430568695
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,4096,768,0.007782400151093801
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,4096,2560,0.019080533583958944
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,4096,2048,0.015735466281572977
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,4096,1024,0.009898666540781658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,4096,256,0.0039594667653242745
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,4096,64,0.0034133332471052804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,4096,128,0.003549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,4096,32,0.002935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3584,7168,0.03863893349965413
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3584,6144,0.031778132915496825
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3584,8192,0.04427093267440796
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3584,10240,0.05130240122477213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3584,12288,0.06109866698582968
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,4096,65536,0.3227306683858236
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3584,16384,0.08253440062204996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3584,5120,0.02744320034980774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3584,3584,0.02249386707941691
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3584,4096,0.02211839954058329
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3584,3072,0.018227199713389076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3584,2560,0.017544533809026083
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3584,1024,0.009215999643007915
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3584,768,0.007031466563542683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3584,512,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3584,1536,0.012117333213488261
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3584,2048,0.014574933052062988
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3584,256,0.008328533172607422
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3584,64,0.0034133332471052804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3584,32,0.002833066632350286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3584,128,0.0033791999022165934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3072,8192,0.03597653309504191
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3072,12288,0.05058559974034628
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3072,7168,0.03174399932225545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3072,10240,0.04341760079065959
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3584,65536,0.28101971944173176
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3072,16384,0.07167999744415283
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3072,6144,0.028194133440653486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3072,5120,0.02461013396581014
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3072,4096,0.020889600118001304
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3072,2048,0.013687466581662497
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3072,2560,0.016622933745384216
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3072,3584,0.020787199338277183
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3072,3072,0.018705066045125326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3072,1024,0.008226133386294047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3072,1536,0.011400533715883891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3072,768,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3072,128,0.0032768001159032187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3072,65536,0.24139092763264974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3072,512,0.004539733131726583
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3072,256,0.00365226666132609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3072,32,0.0027306665976842242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,3072,64,0.0029696000119050344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2560,6144,0.024678399165471397
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2560,10240,0.04041386842727661
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2560,12288,0.043144532044728595
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2560,5120,0.021572266022364298
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2560,7168,0.02775040070215861
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2560,8192,0.031027199824651082
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2560,4096,0.018158932526906334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2560,16384,0.06301013231277466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2560,3072,0.016520532965660095
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2560,3584,0.01648640036582947
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2560,1024,0.007714133461316426
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2560,2560,0.013550933202107748
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2560,2048,0.013960533340771995
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2560,1536,0.011264000336329143
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2560,512,0.004095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2560,768,0.006758399804433187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2560,256,0.0036864000062147772
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2560,128,0.00365226666132609
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2560,64,0.0029696000119050344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2560,65536,0.2075647989908854
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2560,32,0.002833066632350286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2048,10240,0.03853653271993001
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2048,8192,0.02877440055211385
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2048,16384,0.04997119903564453
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2048,12288,0.041915734608968094
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2048,7168,0.029286400477091475
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2048,6144,0.02775040070215861
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2048,3584,0.017169066270192466
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2048,4096,0.018158932526906334
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2048,3072,0.015496533115704855
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2048,5120,0.023176532983779908
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2048,2560,0.013448533415794373
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2048,2048,0.0105813334385554
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2048,1024,0.006587733328342438
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2048,512,0.004676266511281332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2048,1536,0.00942080020904541
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2048,768,0.00580266664425532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2048,256,0.003618133316437403
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2048,65536,0.1646933396657308
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2048,32,0.0030720000465710956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2048,128,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,2048,64,0.00314026673634847
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1536,12288,0.03300693432490031
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1536,10240,0.028433066606521607
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1536,7168,0.021538132429122926
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1536,8192,0.025156267484029132
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1536,16384,0.0425983985265096
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1536,6144,0.01771519978841146
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1536,5120,0.01628159979979197
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1536,4096,0.014711466431617738
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1536,3072,0.011878400047620138
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1536,3584,0.013653332988421122
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1536,65536,0.12342613538106281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1536,2560,0.011059199770291645
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1536,2048,0.008703999718030294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1536,1536,0.0076117331782976794
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1536,768,0.005051733553409576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1536,1024,0.005700266857941946
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1536,512,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1536,128,0.0034815999368826545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1536,256,0.003549866626660029
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1536,64,0.0030720000465710956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1536,32,0.0031061333914597826
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1024,10240,0.01952426632245382
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1024,16384,0.02815999984741211
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1024,12288,0.023244800170262654
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1024,8192,0.016179200013478598
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1024,7168,0.014847999811172486
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1024,6144,0.012834133704503379
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1024,5120,0.011639466881752015
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1024,4096,0.009898666540781658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1024,65536,0.08686933517456055
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1024,3584,0.009523199995358785
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1024,2560,0.00730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1024,2048,0.005870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1024,3072,0.008123733103275299
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1024,1536,0.006826666494210561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1024,1024,0.00477866679430008
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1024,768,0.004676266511281332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1024,128,0.0031744000812371576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1024,512,0.0046079998215039575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1024,256,0.003583999971548716
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1024,32,0.0030037333567937215
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,1024,64,0.002935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,768,12288,0.018090667327245076
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,768,16384,0.02321066657702128
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,768,10240,0.015564800302187601
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,768,7168,0.01225386659304301
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,768,8192,0.013346133629480996
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,768,4096,0.008226133386294047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,768,65536,0.06584320068359376
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,768,5120,0.009898666540781658
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,768,6144,0.010683733224868774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,768,3584,0.007987200220425924
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,768,3072,0.006929066777229309
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,768,2560,0.008601599931716919
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,768,2048,0.006690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,768,1536,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,768,1024,0.005222400029500326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,768,768,0.004266666869322458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,768,128,0.0030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,768,512,0.003857066730658213
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,768,256,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,768,64,0.002935466667016347
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,768,32,0.0030378667016824085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,512,16384,0.018261333306630455
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,512,12288,0.016145066420237223
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,512,10240,0.015530666708946228
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,512,7168,0.011400533715883891
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,512,65536,0.04980053504308064
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,512,8192,0.011571199695269267
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,512,6144,0.009796266754468281
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,512,5120,0.008738133311271667
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,512,4096,0.00730453332265218
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,512,3584,0.007236266632874806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,512,3072,0.006348800162474315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,512,2560,0.0055637334783871974
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,512,2048,0.0054954667886098225
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,512,768,0.004095999896526337
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,512,1024,0.0037546666959921518
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,512,1536,0.004266666869322458
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,512,512,0.004334933559099833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,512,128,0.0030720000465710956
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,512,256,0.0032085334261258446
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,512,32,0.002525866776704788
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,512,64,0.00290133332212766
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,256,16384,0.011025066177050274
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,256,65536,0.026009599367777508
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,256,12288,0.009386666615804036
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,256,8192,0.006894933183987935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,256,10240,0.00846506655216217
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,256,7168,0.006690133114655812
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,256,6144,0.006792533397674561
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,256,5120,0.006041599810123444
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,256,4096,0.006109866499900818
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,256,3584,0.006860800087451935
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,256,2560,0.004437333345413208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,256,1024,0.003345066557327906
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,256,3072,0.004744533201058706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,256,2048,0.004130133241415024
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,256,1536,0.003993600110212962
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,256,768,0.0034815999368826545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,256,512,0.0034133332471052804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,256,256,0.0027647999425729113
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,256,128,0.002833066632350286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,256,64,0.00266239990790685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,128,16384,0.008703999718030294
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,128,65536,0.01672533353169759
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,256,32,0.002628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,128,12288,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,128,10240,0.006724266707897187
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,128,8192,0.00580266664425532
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,128,6144,0.005836800237496694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,128,7168,0.005870933334032694
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,128,5120,0.005666133264700572
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,128,4096,0.006997333467006683
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,128,3072,0.00481279989083608
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,128,3584,0.004846933484077454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,128,2560,0.004300799965858459
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,128,1536,0.0034815999368826545
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,128,1024,0.0031061333914597826
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,128,2048,0.0034133332471052804
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,128,768,0.0033791999022165934
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,128,512,0.003310933212439219
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,128,256,0.002833066632350286
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,128,128,0.002628266563018163
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,128,32,0.00266239990790685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,128,64,0.0025941332181294756
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,64,65536,0.013994666934013366
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,64,16384,0.007202133536338806
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,64,12288,0.006348800162474315
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,64,10240,0.006246399879455566
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,64,8192,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,64,6144,0.005529599885145823
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,64,7168,0.0055978665749231975
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,64,4096,0.0053247998158137
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,64,5120,0.005393066505591074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,64,3584,0.005358933409055074
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,64,3072,0.005256533126036326
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,64,2560,0.005188266436258951
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,64,2048,0.005051733553409576
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,64,1536,0.005085866649945577
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,64,768,0.0045738667249679565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,64,256,0.004437333345413208
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,64,512,0.0044714664419492085
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,64,1024,0.0046079998215039575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,64,128,0.0029696000119050344
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,64,64,0.002696533252795537
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,64,32,0.0027306665976842242
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,32,65536,0.010615467031796774
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,32,16384,0.006451199948787689
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,32,12288,0.006075733403364817
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,32,10240,0.005939200023810069
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,32,7168,0.004915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,32,6144,0.004915200173854828
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,32,8192,0.004949333270390829
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,32,5120,0.004744533201058706
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,32,4096,0.004846933484077454
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,32,3584,0.0045738667249679565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,32,3072,0.004676266511281332
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,32,2560,0.0046079998215039575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,32,2048,0.0046079998215039575
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,32,1536,0.0045738667249679565
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,32,768,0.004369066655635833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,32,512,0.004403199752171834
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,32,1024,0.004369066655635833
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,32,256,0.004164266586303711
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,32,128,0.008260266482830047
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,32,64,0.00266239990790685
TRTLLM,1.0.0,NVIDIA A100-SXM4-80GB,gemm,torch_flow,float16,1,32,32,0.002491733431816101
