108 lines
16 KiB
Plaintext
108 lines
16 KiB
Plaintext
job 17971938 starting on gpub022.delta.ncsa.illinois.edu
|
|
|
|
|
|
test-gpt2-kernels - TF32 Mode is turned OFF.
|
|
test-gpt2 - [State]
|
|
Batch Size: 4
|
|
Sequence Length: 64
|
|
GPT2 - Allocated 221 MiB for activations
|
|
test-gpt2 - Printing 10 Expected vs Actual Logits:
|
|
-43.431618, -43.431740
|
|
-39.836346, -39.836452
|
|
-43.065910, -43.066021
|
|
-42.828045, -42.828144
|
|
-43.529541, -43.529644
|
|
-44.318398, -44.318497
|
|
-41.227425, -41.227528
|
|
-41.270760, -41.270863
|
|
-42.541393, -42.541523
|
|
-42.394997, -42.395119
|
|
---------------- Logits OK! (max err 0.000069, RMSE 0.000002) ----------------
|
|
_________________________ All tests passed! _________________________
|
|
Collecting data...
|
|
Generating '/tmp/nsys-report-9a09.qdstrm'
|
|
|
|
[1/8] [0% ] report_req_4.nsys-rep
|
|
[1/8] [0% ] report_req_4.nsys-rep
|
|
[1/8] [5% ] report_req_4.nsys-rep
|
|
[1/8] [====28% ] report_req_4.nsys-rep
|
|
[1/8] [===========50% ] report_req_4.nsys-rep
|
|
[1/8] [===========51% ] report_req_4.nsys-rep
|
|
[1/8] [======34% ] report_req_4.nsys-rep
|
|
[1/8] [===========51% ] report_req_4.nsys-rep
|
|
[1/8] [===========52% ] report_req_4.nsys-rep
|
|
[1/8] [===========53% ] report_req_4.nsys-rep
|
|
[1/8] [=============59% ] report_req_4.nsys-rep
|
|
[1/8] [==================77% ] report_req_4.nsys-rep
|
|
[1/8] [======================91% ] report_req_4.nsys-rep
|
|
[1/8] [========================100%] report_req_4.nsys-rep
|
|
[1/8] [========================100%] report_req_4.nsys-rep
|
|
|
|
[2/8] [0% ] report_req_4.sqlite
|
|
[2/8] [1% ] report_req_4.sqlite
|
|
[2/8] [2% ] report_req_4.sqlite
|
|
[2/8] [3% ] report_req_4.sqlite
|
|
[2/8] [4% ] report_req_4.sqlite
|
|
[2/8] [5% ] report_req_4.sqlite
|
|
[2/8] [6% ] report_req_4.sqlite
|
|
[2/8] [7% ] report_req_4.sqlite
|
|
[2/8] [8% ] report_req_4.sqlite
|
|
[2/8] [9% ] report_req_4.sqlite
|
|
[2/8] [10% ] report_req_4.sqlite
|
|
[2/8] [11% ] report_req_4.sqlite
|
|
[2/8] [12% ] report_req_4.sqlite
|
|
[2/8] [13% ] report_req_4.sqlite
|
|
[2/8] [14% ] report_req_4.sqlite
|
|
[2/8] [=15% ] report_req_4.sqlite
|
|
[2/8] [=16% ] report_req_4.sqlite
|
|
[2/8] [=17% ] report_req_4.sqlite
|
|
[2/8] [==18% ] report_req_4.sqlite
|
|
[2/8] [==19% ] report_req_4.sqlite
|
|
[2/8] [==20% ] report_req_4.sqlite
|
|
[2/8] [==21% ] report_req_4.sqlite
|
|
[2/8] [===22% ] report_req_4.sqlite
|
|
[2/8] [===23% ] report_req_4.sqlite
|
|
[2/8] [===24% ] report_req_4.sqlite
|
|
[2/8] [====25% ] report_req_4.sqlite
|
|
[2/8] [====26% ] report_req_4.sqlite
|
|
[2/8] [====27% ] report_req_4.sqlite
|
|
[2/8] [====28% ] report_req_4.sqlite
|
|
[2/8] [=====29% ] report_req_4.sqlite
|
|
[2/8] [=====30% ] report_req_4.sqlite
|
|
[2/8] [=====31% ] report_req_4.sqlite
|
|
[2/8] [=====32% ] report_req_4.sqlite
|
|
[2/8] [======33% ] report_req_4.sqlite
|
|
[2/8] [======34% ] report_req_4.sqlite
|
|
[2/8] [======35% ] report_req_4.sqlite
|
|
[2/8] [=======36% ] report_req_4.sqlite
|
|
[2/8] [=======37% ] report_req_4.sqlite
|
|
[2/8] [=======38% ] report_req_4.sqlite
|
|
[2/8] [=======39% ] report_req_4.sqlite
|
|
[2/8] [========40% ] report_req_4.sqlite
|
|
[2/8] [========41% ] report_req_4.sqlite
|
|
[2/8] [========42% ] report_req_4.sqlite
|
|
[2/8] [=========43% ] report_req_4.sqlite
|
|
[2/8] [=========44% ] report_req_4.sqlite
|
|
[2/8] [=========45% ] report_req_4.sqlite
|
|
[2/8] [=========46% ] report_req_4.sqlite
|
|
[2/8] [==========47% ] report_req_4.sqlite
|
|
[2/8] [==========48% ] report_req_4.sqlite
|
|
[2/8] [==========49% ] report_req_4.sqlite
|
|
[2/8] [===========50% ] report_req_4.sqlite
|
|
[2/8] [===========51% ] report_req_4.sqlite
|
|
[2/8] [===========52% ] report_req_4.sqlite
|
|
[2/8] [===========53% ] report_req_4.sqlite
|
|
[2/8] [============54% ] report_req_4.sqlite
|
|
[2/8] [============55% ] report_req_4.sqlite
|
|
[2/8] [============56% ] report_req_4.sqlite
|
|
[2/8] [============57% ] report_req_4.sqlite
|
|
[2/8] [=============58% ] report_req_4.sqlite
|
|
[2/8] [=============59% ] report_req_4.sqlite
|
|
[2/8] [=============60% ] report_req_4.sqlite
|
|
[2/8] [==============61% ] report_req_4.sqlite
|
|
[2/8] [==============62% ] report_req_4.sqlite
|
|
[2/8] [==============63% ] report_req_4.sqlite
|
|
[2/8] [==============64% ] report_req_4.sqlite
|
|
[2/8] [===============65% ] report_req_4.sqlite
|
|
[2/8] [===============66% ] report_req_4.sqlite
|
|
[2/8] [===============67% ] report_req_4.sqlite
|
|
[2/8] [================68% ] report_req_4.sqlite
|
|
[2/8] [================69% ] report_req_4.sqlite
|
|
[2/8] [================70% ] report_req_4.sqlite
|
|
[2/8] [================71% ] report_req_4.sqlite
|
|
[2/8] [=================72% ] report_req_4.sqlite
|
|
[2/8] [=================73% ] report_req_4.sqlite
|
|
[2/8] [=================74% ] report_req_4.sqlite
|
|
[2/8] [==================75% ] report_req_4.sqlite
|
|
[2/8] [==================76% ] report_req_4.sqlite
|
|
[2/8] [==================77% ] report_req_4.sqlite
|
|
[2/8] [==================78% ] report_req_4.sqlite
|
|
[2/8] [===================79% ] report_req_4.sqlite
|
|
[2/8] [===================80% ] report_req_4.sqlite
|
|
[2/8] [===================81% ] report_req_4.sqlite
|
|
[2/8] [===================82% ] report_req_4.sqlite
|
|
[2/8] [====================83% ] report_req_4.sqlite
|
|
[2/8] [====================84% ] report_req_4.sqlite
|
|
[2/8] [====================85% ] report_req_4.sqlite
|
|
[2/8] [=====================86% ] report_req_4.sqlite
|
|
[2/8] [=====================87% ] report_req_4.sqlite
|
|
[2/8] [=====================88% ] report_req_4.sqlite
|
|
[2/8] [=====================89% ] report_req_4.sqlite
|
|
[2/8] [======================90% ] report_req_4.sqlite
|
|
[2/8] [======================91% ] report_req_4.sqlite
|
|
[2/8] [======================92% ] report_req_4.sqlite
|
|
[2/8] [=======================93% ] report_req_4.sqlite
|
|
[2/8] [=======================94% ] report_req_4.sqlite
|
|
[2/8] [=======================95% ] report_req_4.sqlite
|
|
[2/8] [=======================96% ] report_req_4.sqlite
|
|
[2/8] [========================97% ] report_req_4.sqlite
|
|
[2/8] [========================98% ] report_req_4.sqlite
|
|
[2/8] [========================99% ] report_req_4.sqlite
|
|
[2/8] [========================100%] report_req_4.sqlite
|
|
[2/8] [========================100%] report_req_4.sqlite
|
|
[3/8] Executing 'nvtx_sum' stats report
|
|
[4/8] Executing 'osrt_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Num Calls Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name
|
|
-------- --------------- --------- ---------- ---------- -------- --------- ----------- ----------------------
|
|
72.0 706271722 17 41545395.4 21764331.0 2455 105577183 46178357.9 poll
|
|
16.0 157480946 581 271051.5 23474.0 1022 21596356 1265979.2 ioctl
|
|
9.7 95519940 6 15919990.0 69450.0 7424 87023730 34991208.6 fread
|
|
0.7 7116288 1 7116288.0 7116288.0 7116288 7116288 0.0 pthread_cond_broadcast
|
|
0.7 6683027 1 6683027.0 6683027.0 6683027 6683027 0.0 pthread_cond_wait
|
|
0.2 2395073 25 95802.9 8977.0 7164 1672329 331103.1 mmap64
|
|
0.2 2118124 11 192556.7 27401.0 1062 855742 300169.0 write
|
|
0.1 1406722 38 37019.0 6271.5 3787 518551 104526.5 fopen
|
|
0.1 660508 32 20640.9 2259.0 1383 314419 71634.1 fclose
|
|
0.1 601947 17 35408.6 4919.0 2525 384280 91907.4 mmap
|
|
0.0 360014 2 180007.0 180007.0 167553 192461 17612.6 pthread_create
|
|
0.0 336551 55 6119.1 6021.0 2394 16411 2749.7 open64
|
|
0.0 212838 1 212838.0 212838.0 212838 212838 0.0 fflush
|
|
0.0 78526 1 78526.0 78526.0 78526 78526 0.0 fgets
|
|
0.0 41037 8 5129.6 4934.5 3036 7744 1719.8 munmap
|
|
0.0 38151 6 6358.5 6477.0 3276 9107 2070.2 open
|
|
0.0 22041 3 7347.0 9007.0 3927 9107 2962.2 pipe2
|
|
0.0 21344 12 1778.7 1213.0 1022 3978 999.8 read
|
|
0.0 21000 1 21000.0 21000.0 21000 21000 0.0 connect
|
|
0.0 20027 2 10013.5 10013.5 8365 11662 2331.3 socket
|
|
0.0 6553 1 6553.0 6553.0 6553 6553 0.0 fopen64
|
|
0.0 3175 2 1587.5 1587.5 1523 1652 91.2 fwrite
|
|
0.0 2415 1 2415.0 2415.0 2415 2415 0.0 bind
|
|
0.0 2125 2 1062.5 1062.5 1032 1093 43.1 fcntl
|
|
|
|
[5/8] Executing 'cuda_api_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Num Calls Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name
|
|
-------- --------------- --------- ---------- ---------- -------- --------- ----------- ---------------------------------
|
|
83.6 293033557 3 97677852.3 36393158.0 19847 256620552 138844664.2 cudaMemcpy
|
|
8.4 29603500 147 201384.4 2815.0 2685 28629047 2361040.2 cudaLaunchKernel
|
|
4.5 15916639 6 2652773.2 122850.5 3066 15473999 6281511.1 cudaMalloc
|
|
1.5 5351312 12 445942.7 8245.5 201 1942786 645982.5 cudaFree
|
|
0.7 2445367 1 2445367.0 2445367.0 2445367 2445367 0.0 cudaFreeHost
|
|
0.6 1973533 1 1973533.0 1973533.0 1973533 1973533 0.0 cudaGetDeviceProperties_v2_v12000
|
|
0.5 1862896 1 1862896.0 1862896.0 1862896 1862896 0.0 cudaMallocHost
|
|
0.1 192866 838 230.2 191.0 90 7394 351.2 cuGetProcAddress_v2
|
|
0.0 37921 1 37921.0 37921.0 37921 37921 0.0 cudaMemset
|
|
0.0 12805 4 3201.3 2419.5 1223 6743 2437.3 cudaDeviceSynchronize
|
|
0.0 11563 18 642.4 300.5 270 2083 678.8 cudaEventCreateWithFlags
|
|
0.0 7813 18 434.1 315.0 230 2214 459.8 cudaEventDestroy
|
|
0.0 5469 4 1367.3 1367.0 1232 1503 116.6 cuInit
|
|
0.0 1773 3 591.0 130.0 110 1533 815.9 cuModuleGetLoadingMode
|
|
0.0 942 2 471.0 471.0 260 682 298.4 cudaGetDriverEntryPoint_v11030
|
|
|
|
[6/8] Executing 'cuda_gpu_kern_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Instances Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name
|
|
-------- --------------- --------- --------- --------- -------- -------- ----------- ----------------------------------------------------------------------------------------------------
|
|
97.3 243465041 49 4968674.3 4541726.0 1227616 73781940 10137437.2 matmul_forward_kernel(float *, const float *, const float *, const float *, int, int, int, int)
|
|
1.4 3574684 25 142987.4 142944.0 141408 147104 1126.4 layernorm_forward_kernel(float *, float *, float *, const float *, const float *, const float *, in…
|
|
1.0 2583647 12 215303.9 215328.0 214879 215392 136.1 void flash_attention_forward_kernel<(int)64, (int)64, (int)64>(const float *, const float *, const …
|
|
0.1 227840 12 18986.7 19008.0 18688 19136 115.2 gelu_forward_kernel(float *, const float *, int)
|
|
0.1 146144 24 6089.3 6128.0 5696 6464 346.6 residual_forward_kernel(float *, const float *, const float *, int)
|
|
0.0 79776 12 6648.0 6640.0 6464 6816 119.1 permute_kernel(float *, float *, float *, const float *, int, int, int, int)
|
|
0.0 79296 1 79296.0 79296.0 79296 79296 0.0 encoder_forward_kernel(float *, const int *, const float *, const float *, int, int, int)
|
|
0.0 36640 12 3053.3 3040.0 3040 3104 21.4 unpermute_kernel(const float *, float *, int, int, int, int)
|
|
|
|
[7/8] Executing 'cuda_gpu_mem_time_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Count Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Operation
|
|
-------- --------------- ----- ---------- ---------- -------- -------- ----------- ----------------------------
|
|
82.7 36306187 2 18153093.5 18153093.5 832 36305355 25671174.4 [CUDA memcpy Host-to-Device]
|
|
16.4 7213212 1 7213212.0 7213212.0 7213212 7213212 0.0 [CUDA memcpy Device-to-Host]
|
|
0.9 407200 1 407200.0 407200.0 407200 407200 0.0 [CUDA memset]
|
|
|
|
[8/8] Executing 'cuda_gpu_mem_size_sum' stats report
|
|
|
|
Total (MB) Count Avg (MB) Med (MB) Min (MB) Max (MB) StdDev (MB) Operation
|
|
---------- ----- -------- -------- -------- -------- ----------- ----------------------------
|
|
497.905 2 248.952 248.952 0.001 497.904 352.070 [CUDA memcpy Host-to-Device]
|
|
232.392 1 232.392 232.392 232.392 232.392 0.000 [CUDA memset]
|
|
51.511 1 51.511 51.511 51.511 51.511 0.000 [CUDA memcpy Device-to-Host]
|
|
|
|
Generated:
|
|
/u/asingh41/sp26_ece408_Temperature1.5/output/perf/reports/report_req_4.nsys-rep
|
|
/u/asingh41/sp26_ece408_Temperature1.5/output/perf/reports/report_req_4.sqlite
|