117 lines
17 KiB
Plaintext
117 lines
17 KiB
Plaintext
job 18004586 starting on gpub041.delta.ncsa.illinois.edu
|
|
|
|
|
|
test-gpt2-kernels - TF32 Mode is turned OFF.
|
|
test-gpt2 - [State]
|
|
Batch Size: 4
|
|
Sequence Length: 64
|
|
GPT2 - Allocated 221 MiB for activations
|
|
test-gpt2 - Printing 10 Expected vs Actual Logits:
|
|
-43.431618, -43.315117
|
|
-39.836346, -39.732384
|
|
-43.065910, -42.960098
|
|
-42.828045, -42.723515
|
|
-43.529541, -43.418613
|
|
-44.318398, -44.210033
|
|
-41.227425, -41.119377
|
|
-41.270760, -41.164490
|
|
-42.541393, -42.428894
|
|
-42.394997, -42.288929
|
|
---------------- Logits OK! (max err 0.082334, RMSE 0.002728) ----------------
|
|
_________________________ All tests passed! _________________________
|
|
Collecting data...
|
|
Generating '/tmp/nsys-report-cacf.qdstrm'
|
|
|
|
[1/8] [0% ] report_op_16.nsys-rep
|
|
[1/8] [0% ] report_op_16.nsys-rep
|
|
[1/8] [====27% ] report_op_16.nsys-rep
|
|
[1/8] [====28% ] report_op_16.nsys-rep
|
|
[1/8] [=====29% ] report_op_16.nsys-rep
|
|
[1/8] [=====30% ] report_op_16.nsys-rep
|
|
[1/8] [=======38% ] report_op_16.nsys-rep
|
|
[1/8] [=========46% ] report_op_16.nsys-rep
|
|
[1/8] [==========47% ] report_op_16.nsys-rep
|
|
[1/8] [==========48% ] report_op_16.nsys-rep
|
|
[1/8] [==========49% ] report_op_16.nsys-rep
|
|
[1/8] [============55% ] report_op_16.nsys-rep
|
|
[1/8] [============56% ] report_op_16.nsys-rep
|
|
[1/8] [============57% ] report_op_16.nsys-rep
|
|
[1/8] [=============58% ] report_op_16.nsys-rep
|
|
[1/8] [==================77% ] report_op_16.nsys-rep
|
|
[1/8] [=======================94% ] report_op_16.nsys-rep
|
|
[1/8] [========================100%] report_op_16.nsys-rep
|
|
[1/8] [========================100%] report_op_16.nsys-rep
|
|
|
|
[2/8] [0% ] report_op_16.sqlite
|
|
[2/8] [1% ] report_op_16.sqlite
|
|
[2/8] [2% ] report_op_16.sqlite
|
|
[2/8] [3% ] report_op_16.sqlite
|
|
[2/8] [4% ] report_op_16.sqlite
|
|
[2/8] [5% ] report_op_16.sqlite
|
|
[2/8] [6% ] report_op_16.sqlite
|
|
[2/8] [7% ] report_op_16.sqlite
|
|
[2/8] [8% ] report_op_16.sqlite
|
|
[2/8] [9% ] report_op_16.sqlite
|
|
[2/8] [10% ] report_op_16.sqlite
|
|
[2/8] [11% ] report_op_16.sqlite
|
|
[2/8] [12% ] report_op_16.sqlite
|
|
[2/8] [13% ] report_op_16.sqlite
|
|
[2/8] [14% ] report_op_16.sqlite
|
|
[2/8] [=15% ] report_op_16.sqlite
|
|
[2/8] [=16% ] report_op_16.sqlite
|
|
[2/8] [=17% ] report_op_16.sqlite
|
|
[2/8] [==18% ] report_op_16.sqlite
|
|
[2/8] [==19% ] report_op_16.sqlite
|
|
[2/8] [==20% ] report_op_16.sqlite
|
|
[2/8] [==21% ] report_op_16.sqlite
|
|
[2/8] [===22% ] report_op_16.sqlite
|
|
[2/8] [===23% ] report_op_16.sqlite
|
|
[2/8] [===24% ] report_op_16.sqlite
|
|
[2/8] [====25% ] report_op_16.sqlite
|
|
[2/8] [====26% ] report_op_16.sqlite
|
|
[2/8] [====27% ] report_op_16.sqlite
|
|
[2/8] [====28% ] report_op_16.sqlite
|
|
[2/8] [=====29% ] report_op_16.sqlite
|
|
[2/8] [=====30% ] report_op_16.sqlite
|
|
[2/8] [=====31% ] report_op_16.sqlite
|
|
[2/8] [=====32% ] report_op_16.sqlite
|
|
[2/8] [======33% ] report_op_16.sqlite
|
|
[2/8] [======34% ] report_op_16.sqlite
|
|
[2/8] [======35% ] report_op_16.sqlite
|
|
[2/8] [=======36% ] report_op_16.sqlite
|
|
[2/8] [=======37% ] report_op_16.sqlite
|
|
[2/8] [=======38% ] report_op_16.sqlite
|
|
[2/8] [=======39% ] report_op_16.sqlite
|
|
[2/8] [========40% ] report_op_16.sqlite
|
|
[2/8] [========41% ] report_op_16.sqlite
|
|
[2/8] [========42% ] report_op_16.sqlite
|
|
[2/8] [=========43% ] report_op_16.sqlite
|
|
[2/8] [=========44% ] report_op_16.sqlite
|
|
[2/8] [=========45% ] report_op_16.sqlite
|
|
[2/8] [=========46% ] report_op_16.sqlite
|
|
[2/8] [==========47% ] report_op_16.sqlite
|
|
[2/8] [==========48% ] report_op_16.sqlite
|
|
[2/8] [==========49% ] report_op_16.sqlite
|
|
[2/8] [===========50% ] report_op_16.sqlite
|
|
[2/8] [===========51% ] report_op_16.sqlite
|
|
[2/8] [===========52% ] report_op_16.sqlite
|
|
[2/8] [===========53% ] report_op_16.sqlite
|
|
[2/8] [============54% ] report_op_16.sqlite
|
|
[2/8] [============55% ] report_op_16.sqlite
|
|
[2/8] [============56% ] report_op_16.sqlite
|
|
[2/8] [============57% ] report_op_16.sqlite
|
|
[2/8] [=============58% ] report_op_16.sqlite
|
|
[2/8] [=============59% ] report_op_16.sqlite
|
|
[2/8] [=============60% ] report_op_16.sqlite
|
|
[2/8] [==============61% ] report_op_16.sqlite
|
|
[2/8] [==============62% ] report_op_16.sqlite
|
|
[2/8] [==============63% ] report_op_16.sqlite
|
|
[2/8] [==============64% ] report_op_16.sqlite
|
|
[2/8] [===============65% ] report_op_16.sqlite
|
|
[2/8] [===============66% ] report_op_16.sqlite
|
|
[2/8] [===============67% ] report_op_16.sqlite
|
|
[2/8] [================68% ] report_op_16.sqlite
|
|
[2/8] [================69% ] report_op_16.sqlite
|
|
[2/8] [================70% ] report_op_16.sqlite
|
|
[2/8] [================71% ] report_op_16.sqlite
|
|
[2/8] [=================72% ] report_op_16.sqlite
|
|
[2/8] [=================73% ] report_op_16.sqlite
|
|
[2/8] [=================74% ] report_op_16.sqlite
|
|
[2/8] [==================75% ] report_op_16.sqlite
|
|
[2/8] [==================76% ] report_op_16.sqlite
|
|
[2/8] [==================77% ] report_op_16.sqlite
|
|
[2/8] [==================78% ] report_op_16.sqlite
|
|
[2/8] [===================79% ] report_op_16.sqlite
|
|
[2/8] [===================80% ] report_op_16.sqlite
|
|
[2/8] [===================81% ] report_op_16.sqlite
|
|
[2/8] [===================82% ] report_op_16.sqlite
|
|
[2/8] [====================83% ] report_op_16.sqlite
|
|
[2/8] [====================84% ] report_op_16.sqlite
|
|
[2/8] [====================85% ] report_op_16.sqlite
|
|
[2/8] [=====================86% ] report_op_16.sqlite
|
|
[2/8] [=====================87% ] report_op_16.sqlite
|
|
[2/8] [=====================88% ] report_op_16.sqlite
|
|
[2/8] [=====================89% ] report_op_16.sqlite
|
|
[2/8] [======================90% ] report_op_16.sqlite
|
|
[2/8] [======================91% ] report_op_16.sqlite
|
|
[2/8] [======================92% ] report_op_16.sqlite
|
|
[2/8] [=======================93% ] report_op_16.sqlite
|
|
[2/8] [=======================94% ] report_op_16.sqlite
|
|
[2/8] [=======================95% ] report_op_16.sqlite
|
|
[2/8] [=======================96% ] report_op_16.sqlite
|
|
[2/8] [========================97% ] report_op_16.sqlite
|
|
[2/8] [========================98% ] report_op_16.sqlite
|
|
[2/8] [========================99% ] report_op_16.sqlite
|
|
[2/8] [========================100%] report_op_16.sqlite
|
|
[2/8] [========================100%] report_op_16.sqlite
|
|
[3/8] Executing 'nvtx_sum' stats report
|
|
[4/8] Executing 'osrt_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Num Calls Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name
|
|
-------- --------------- --------- ----------- ---------- -------- --------- ----------- --------------
|
|
58.3 1107473464 21 52736831.6 32614355.0 1322 102246365 47278143.8 poll
|
|
32.4 615776283 6 102629380.5 9779477.0 6893 526459513 209303008.5 fread
|
|
8.7 164992616 592 278703.7 23008.0 1041 22196543 1294110.2 ioctl
|
|
0.2 4255919 38 111997.9 6477.0 3637 3134408 512054.3 fopen
|
|
0.1 2359811 25 94392.4 8766.0 7474 1634199 323598.5 mmap64
|
|
0.1 999815 32 31244.2 2119.0 1333 525704 114089.3 fclose
|
|
0.0 836867 11 76078.8 29566.0 1603 467756 132907.7 write
|
|
0.0 568205 17 33423.8 8155.0 1943 376916 90182.7 mmap
|
|
0.0 345457 2 172728.5 172728.5 163176 182281 13509.3 pthread_create
|
|
0.0 328243 55 5968.1 5862.0 2344 13786 2471.7 open64
|
|
0.0 223538 1 223538.0 223538.0 223538 223538 0.0 fflush
|
|
0.0 80441 1 80441.0 80441.0 80441 80441 0.0 fgets
|
|
0.0 46428 9 5158.7 4038.0 2575 9789 2506.9 munmap
|
|
0.0 37720 6 6286.7 6542.5 3276 8926 1948.3 open
|
|
0.0 27181 3 9060.3 8235.0 6613 12333 2948.0 pipe2
|
|
0.0 19594 11 1781.3 1282.0 1002 3506 893.6 read
|
|
0.0 17683 2 8841.5 8841.5 5851 11832 4229.2 socket
|
|
0.0 10981 1 10981.0 10981.0 10981 10981 0.0 connect
|
|
0.0 6862 1 6862.0 6862.0 6862 6862 0.0 fopen64
|
|
0.0 4558 2 2279.0 2279.0 2074 2484 289.9 fwrite
|
|
0.0 2174 1 2174.0 2174.0 2174 2174 0.0 bind
|
|
0.0 1062 1 1062.0 1062.0 1062 1062 0.0 listen
|
|
|
|
[5/8] Executing 'cuda_api_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Num Calls Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name
|
|
-------- --------------- --------- ---------- ---------- -------- -------- ----------- ---------------------------------
|
|
54.2 51711904 3 17237301.3 15706111.0 19487 35986306 18032232.9 cudaMemcpy
|
|
22.1 21075926 12 1756327.2 15158.5 190 16493445 4691526.2 cudaFree
|
|
10.2 9764437 4 2441109.3 2908985.5 877652 3068814 1045305.5 cuLibraryLoadData
|
|
3.8 3644594 207 17606.7 4519.0 2725 2467218 171275.3 cudaLaunchKernel
|
|
2.5 2409812 1 2409812.0 2409812.0 2409812 2409812 0.0 cudaFreeHost
|
|
1.7 1608458 18 89358.8 305.5 260 1597961 376498.2 cudaEventCreateWithFlags
|
|
1.4 1351740 1 1351740.0 1351740.0 1351740 1351740 0.0 cudaMallocHost
|
|
1.4 1315042 1 1315042.0 1315042.0 1315042 1315042 0.0 cudaGetDeviceProperties_v2_v12000
|
|
1.1 1038424 6 173070.7 135794.0 4840 445143 174111.4 cudaMalloc
|
|
1.0 977961 4 244490.3 219145.5 1072 538598 283164.8 cuLibraryGetKernel
|
|
0.2 229567 838 273.9 180.0 90 48441 1670.5 cuGetProcAddress_v2
|
|
0.1 137239 36 3812.2 3146.0 2094 14667 2165.3 cudaMemsetAsync
|
|
0.1 96319 36 2675.5 2049.0 1833 7945 1134.7 cudaEventRecord
|
|
0.0 44693 1 44693.0 44693.0 44693 44693 0.0 cudaMemset
|
|
0.0 38138 180 211.9 130.0 70 12083 894.1 cuStreamGetCaptureInfo_v2
|
|
0.0 21040 4 5260.0 3141.0 1503 13255 5424.7 cudaDeviceSynchronize
|
|
0.0 9043 18 502.4 275.5 200 3386 741.5 cudaEventDestroy
|
|
0.0 5723 4 1430.8 1443.5 1163 1673 221.4 cuInit
|
|
0.0 3677 1 3677.0 3677.0 3677 3677 0.0 cudaEventQuery
|
|
0.0 1714 3 571.3 140.0 111 1463 772.3 cuModuleGetLoadingMode
|
|
0.0 941 2 470.5 470.5 300 641 241.1 cudaGetDriverEntryPoint_v11030
|
|
|
|
[6/8] Executing 'cuda_gpu_kern_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Instances Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name
|
|
-------- --------------- --------- -------- -------- -------- -------- ----------- ----------------------------------------------------------------------------------------------------
|
|
35.8 3686110 25 147444.4 147616.0 145600 149216 1035.4 layernorm_forward_kernel(float *, float *, float *, const float *, const float *, const float *, in…
|
|
16.2 1663328 13 127948.3 59392.0 58912 950752 247221.8 ampere_sgemm_128x64_tn
|
|
14.9 1536224 12 128018.7 128112.0 125600 129472 1083.3 ampere_sgemm_64x64_tn
|
|
11.1 1146944 12 95578.7 95616.0 94720 95904 297.0 preattention_scores_kernel(const float *, const float *, int, int, int, int, float *)
|
|
6.5 670624 12 55885.3 55840.0 55136 56928 530.5 void cutlass::Kernel<cutlass::gemm::kernel::Gemm<cutlass::gemm::threadblock::MmaPipelined<cutlass::…
|
|
3.9 398208 12 33184.0 33184.0 32928 33344 111.7 ampere_sgemm_64x32_sliced1x4_tn
|
|
3.3 335424 12 27952.0 27888.0 27296 28672 440.4 attention_v_kernel(float *, const float *, const float *, int, int, int, int)
|
|
2.7 279648 48 5826.0 4320.0 2848 12448 3668.6 bias_add_filler_kernel(float *, const float *, int, int)
|
|
2.0 205856 12 17154.7 17136.0 17024 17312 83.5 softmax_forward_kernel(float *, float, const float *, int, int)
|
|
1.4 141824 24 5909.3 5936.0 5600 6240 246.7 residual_forward_kernel(float *, const float *, const float *, int)
|
|
0.8 80928 1 80928.0 80928.0 80928 80928 0.0 encoder_forward_kernel(float *, const int *, const float *, const float *, int, int, int)
|
|
0.7 76896 12 6408.0 6384.0 6304 6592 84.3 permute_kernel(float *, float *, float *, const float *, int, int, int, int)
|
|
0.7 70688 12 5890.7 5888.0 5824 5952 37.3 unpermute_kernel(const float *, float *, int, int, int, int)
|
|
|
|
[7/8] Executing 'cuda_gpu_mem_time_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Count Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Operation
|
|
-------- --------------- ----- ---------- ---------- -------- -------- ----------- ----------------------------
|
|
82.7 35891318 2 17945659.0 17945659.0 864 35890454 25377772.5 [CUDA memcpy Host-to-Device]
|
|
16.3 7061278 1 7061278.0 7061278.0 7061278 7061278 0.0 [CUDA memcpy Device-to-Host]
|
|
1.0 454592 37 12286.3 1184.0 480 408320 66917.2 [CUDA memset]
|
|
|
|
[8/8] Executing 'cuda_gpu_mem_size_sum' stats report
|
|
|
|
Total (MB) Count Avg (MB) Med (MB) Min (MB) Max (MB) StdDev (MB) Operation
|
|
---------- ----- -------- -------- -------- -------- ----------- ----------------------------
|
|
497.905 2 248.952 248.952 0.001 497.904 352.070 [CUDA memcpy Host-to-Device]
|
|
232.402 37 6.281 0.000 0.000 232.392 38.205 [CUDA memset]
|
|
51.511 1 51.511 51.511 51.511 51.511 0.000 [CUDA memcpy Device-to-Host]
|
|
|
|
Generated:
|
|
/u/asingh41/sp26_ece408_Temperature1.5/output/perf/reports/report_op_16.nsys-rep
|
|
/u/asingh41/sp26_ece408_Temperature1.5/output/perf/reports/report_op_16.sqlite
|