110 lines
17 KiB
Plaintext
110 lines
17 KiB
Plaintext
job 18002391 starting on gpub073.delta.ncsa.illinois.edu
|
|
|
|
|
|
test-gpt2-kernels - TF32 Mode is turned OFF.
|
|
test-gpt2 - [State]
|
|
Batch Size: 4
|
|
Sequence Length: 64
|
|
GPT2 - Allocated 221 MiB for activations
|
|
test-gpt2 - Printing 10 Expected vs Actual Logits:
|
|
-43.431618, -43.431793
|
|
-39.836346, -39.836502
|
|
-43.065910, -43.066078
|
|
-42.828045, -42.828197
|
|
-43.529541, -43.529701
|
|
-44.318398, -44.318558
|
|
-41.227425, -41.227592
|
|
-41.270760, -41.270931
|
|
-42.541393, -42.541584
|
|
-42.394997, -42.395176
|
|
---------------- Logits OK! (max err 0.000050, RMSE 0.000002) ----------------
|
|
_________________________ All tests passed! _________________________
|
|
Collecting data...
|
|
Generating '/tmp/nsys-report-c6cc.qdstrm'
|
|
|
|
[1/8] [0% ] nsys-report-d644.nsys-rep
|
|
[1/8] [0% ] nsys-report-d644.nsys-rep
|
|
[1/8] [====26% ] nsys-report-d644.nsys-rep
|
|
[1/8] [==========49% ] nsys-report-d644.nsys-rep
|
|
[1/8] [===========53% ] nsys-report-d644.nsys-rep
|
|
[1/8] [=======36% ] nsys-report-d644.nsys-rep
|
|
[1/8] [===========52% ] nsys-report-d644.nsys-rep
|
|
[1/8] [============54% ] nsys-report-d644.nsys-rep
|
|
[1/8] [============55% ] nsys-report-d644.nsys-rep
|
|
[1/8] [============56% ] nsys-report-d644.nsys-rep
|
|
[1/8] [==============62% ] nsys-report-d644.nsys-rep
|
|
[1/8] [==================78% ] nsys-report-d644.nsys-rep
|
|
[1/8] [======================92% ] nsys-report-d644.nsys-rep
|
|
[1/8] [========================100%] nsys-report-d644.nsys-rep
|
|
[1/8] [========================100%] nsys-report-d644.nsys-rep
|
|
|
|
[2/8] [0% ] nsys-report-9b06.sqlite
|
|
[2/8] [1% ] nsys-report-9b06.sqlite
|
|
[2/8] [2% ] nsys-report-9b06.sqlite
|
|
[2/8] [3% ] nsys-report-9b06.sqlite
|
|
[2/8] [4% ] nsys-report-9b06.sqlite
|
|
[2/8] [5% ] nsys-report-9b06.sqlite
|
|
[2/8] [6% ] nsys-report-9b06.sqlite
|
|
[2/8] [7% ] nsys-report-9b06.sqlite
|
|
[2/8] [8% ] nsys-report-9b06.sqlite
|
|
[2/8] [9% ] nsys-report-9b06.sqlite
|
|
[2/8] [10% ] nsys-report-9b06.sqlite
|
|
[2/8] [11% ] nsys-report-9b06.sqlite
|
|
[2/8] [12% ] nsys-report-9b06.sqlite
|
|
[2/8] [13% ] nsys-report-9b06.sqlite
|
|
[2/8] [14% ] nsys-report-9b06.sqlite
|
|
[2/8] [=15% ] nsys-report-9b06.sqlite
|
|
[2/8] [=16% ] nsys-report-9b06.sqlite
|
|
[2/8] [=17% ] nsys-report-9b06.sqlite
|
|
[2/8] [==18% ] nsys-report-9b06.sqlite
|
|
[2/8] [==19% ] nsys-report-9b06.sqlite
|
|
[2/8] [==20% ] nsys-report-9b06.sqlite
|
|
[2/8] [==21% ] nsys-report-9b06.sqlite
|
|
[2/8] [===22% ] nsys-report-9b06.sqlite
|
|
[2/8] [===23% ] nsys-report-9b06.sqlite
|
|
[2/8] [===24% ] nsys-report-9b06.sqlite
|
|
[2/8] [====25% ] nsys-report-9b06.sqlite
|
|
[2/8] [====26% ] nsys-report-9b06.sqlite
|
|
[2/8] [====27% ] nsys-report-9b06.sqlite
|
|
[2/8] [====28% ] nsys-report-9b06.sqlite
|
|
[2/8] [=====29% ] nsys-report-9b06.sqlite
|
|
[2/8] [=====30% ] nsys-report-9b06.sqlite
|
|
[2/8] [=====31% ] nsys-report-9b06.sqlite
|
|
[2/8] [=====32% ] nsys-report-9b06.sqlite
|
|
[2/8] [======33% ] nsys-report-9b06.sqlite
|
|
[2/8] [======34% ] nsys-report-9b06.sqlite
|
|
[2/8] [======35% ] nsys-report-9b06.sqlite
|
|
[2/8] [=======36% ] nsys-report-9b06.sqlite
|
|
[2/8] [=======37% ] nsys-report-9b06.sqlite
|
|
[2/8] [=======38% ] nsys-report-9b06.sqlite
|
|
[2/8] [=======39% ] nsys-report-9b06.sqlite
|
|
[2/8] [========40% ] nsys-report-9b06.sqlite
|
|
[2/8] [========41% ] nsys-report-9b06.sqlite
|
|
[2/8] [========42% ] nsys-report-9b06.sqlite
|
|
[2/8] [=========43% ] nsys-report-9b06.sqlite
|
|
[2/8] [=========44% ] nsys-report-9b06.sqlite
|
|
[2/8] [=========45% ] nsys-report-9b06.sqlite
|
|
[2/8] [=========46% ] nsys-report-9b06.sqlite
|
|
[2/8] [==========47% ] nsys-report-9b06.sqlite
|
|
[2/8] [==========48% ] nsys-report-9b06.sqlite
|
|
[2/8] [==========49% ] nsys-report-9b06.sqlite
|
|
[2/8] [===========50% ] nsys-report-9b06.sqlite
|
|
[2/8] [===========51% ] nsys-report-9b06.sqlite
|
|
[2/8] [===========52% ] nsys-report-9b06.sqlite
|
|
[2/8] [===========53% ] nsys-report-9b06.sqlite
|
|
[2/8] [============54% ] nsys-report-9b06.sqlite
|
|
[2/8] [============55% ] nsys-report-9b06.sqlite
|
|
[2/8] [============56% ] nsys-report-9b06.sqlite
|
|
[2/8] [============57% ] nsys-report-9b06.sqlite
|
|
[2/8] [=============58% ] nsys-report-9b06.sqlite
|
|
[2/8] [=============59% ] nsys-report-9b06.sqlite
|
|
[2/8] [=============60% ] nsys-report-9b06.sqlite
|
|
[2/8] [==============61% ] nsys-report-9b06.sqlite
|
|
[2/8] [==============62% ] nsys-report-9b06.sqlite
|
|
[2/8] [==============63% ] nsys-report-9b06.sqlite
|
|
[2/8] [==============64% ] nsys-report-9b06.sqlite
|
|
[2/8] [===============65% ] nsys-report-9b06.sqlite
|
|
[2/8] [===============66% ] nsys-report-9b06.sqlite
|
|
[2/8] [===============67% ] nsys-report-9b06.sqlite
|
|
[2/8] [================68% ] nsys-report-9b06.sqlite
|
|
[2/8] [================69% ] nsys-report-9b06.sqlite
|
|
[2/8] [================70% ] nsys-report-9b06.sqlite
|
|
[2/8] [================71% ] nsys-report-9b06.sqlite
|
|
[2/8] [=================72% ] nsys-report-9b06.sqlite
|
|
[2/8] [=================73% ] nsys-report-9b06.sqlite
|
|
[2/8] [=================74% ] nsys-report-9b06.sqlite
|
|
[2/8] [==================75% ] nsys-report-9b06.sqlite
|
|
[2/8] [==================76% ] nsys-report-9b06.sqlite
|
|
[2/8] [==================77% ] nsys-report-9b06.sqlite
|
|
[2/8] [==================78% ] nsys-report-9b06.sqlite
|
|
[2/8] [===================79% ] nsys-report-9b06.sqlite
|
|
[2/8] [===================80% ] nsys-report-9b06.sqlite
|
|
[2/8] [===================81% ] nsys-report-9b06.sqlite
|
|
[2/8] [===================82% ] nsys-report-9b06.sqlite
|
|
[2/8] [====================83% ] nsys-report-9b06.sqlite
|
|
[2/8] [====================84% ] nsys-report-9b06.sqlite
|
|
[2/8] [====================85% ] nsys-report-9b06.sqlite
|
|
[2/8] [=====================86% ] nsys-report-9b06.sqlite
|
|
[2/8] [=====================87% ] nsys-report-9b06.sqlite
|
|
[2/8] [=====================88% ] nsys-report-9b06.sqlite
|
|
[2/8] [=====================89% ] nsys-report-9b06.sqlite
|
|
[2/8] [======================90% ] nsys-report-9b06.sqlite
|
|
[2/8] [======================91% ] nsys-report-9b06.sqlite
|
|
[2/8] [======================92% ] nsys-report-9b06.sqlite
|
|
[2/8] [=======================93% ] nsys-report-9b06.sqlite
|
|
[2/8] [=======================94% ] nsys-report-9b06.sqlite
|
|
[2/8] [=======================95% ] nsys-report-9b06.sqlite
|
|
[2/8] [=======================96% ] nsys-report-9b06.sqlite
|
|
[2/8] [========================97% ] nsys-report-9b06.sqlite
|
|
[2/8] [========================98% ] nsys-report-9b06.sqlite
|
|
[2/8] [========================99% ] nsys-report-9b06.sqlite
|
|
[2/8] [========================100%] nsys-report-9b06.sqlite
|
|
[2/8] [========================100%] nsys-report-9b06.sqlite
|
|
[3/8] Executing 'nvtx_sum' stats report
|
|
[4/8] Executing 'osrt_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Num Calls Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name
|
|
-------- --------------- --------- ---------- ---------- -------- --------- ----------- ----------------------
|
|
61.1 737068265 17 43356956.8 31803265.0 2204 136058155 46033325.4 poll
|
|
29.3 352852419 581 607319.1 280456.0 1082 20080933 1246338.6 ioctl
|
|
7.2 86629708 6 14438284.7 75346.0 5560 78483379 31535650.3 fread
|
|
1.1 13589855 1 13589855.0 13589855.0 13589855 13589855 0.0 pthread_cond_wait
|
|
0.6 7448723 1 7448723.0 7448723.0 7448723 7448723 0.0 pthread_cond_broadcast
|
|
0.2 2347161 25 93886.4 9638.0 7675 1629425 322632.7 mmap64
|
|
0.2 2110733 32 65960.4 2189.0 1352 1703774 304577.4 fclose
|
|
0.1 1693364 38 44562.2 6256.0 3587 809497 144162.0 fopen
|
|
0.1 746287 11 67844.3 27722.0 1132 318406 94484.9 write
|
|
0.0 594968 17 34998.1 4739.0 2575 402164 96189.6 mmap
|
|
0.0 361558 2 180779.0 180779.0 164709 196849 22726.4 pthread_create
|
|
0.0 353489 55 6427.1 6332.0 2344 13415 2698.2 open64
|
|
0.0 221045 1 221045.0 221045.0 221045 221045 0.0 fflush
|
|
0.0 76323 1 76323.0 76323.0 76323 76323 0.0 fgets
|
|
0.0 37250 8 4656.3 4112.5 2875 7805 1734.3 munmap
|
|
0.0 36069 6 6011.5 5996.5 3086 9027 2010.2 open
|
|
0.0 20939 3 6979.7 8526.0 3727 8686 2818.0 pipe2
|
|
0.0 18773 9 2085.9 2083.0 1082 3727 979.8 read
|
|
0.0 17263 2 8631.5 8631.5 6523 10740 2981.9 socket
|
|
0.0 9477 1 9477.0 9477.0 9477 9477 0.0 connect
|
|
0.0 6272 1 6272.0 6272.0 6272 6272 0.0 fopen64
|
|
0.0 4249 2 2124.5 2124.5 1413 2836 1006.2 fwrite
|
|
0.0 2134 1 2134.0 2134.0 2134 2134 0.0 bind
|
|
0.0 2054 2 1027.0 1027.0 1012 1042 21.2 fcntl
|
|
|
|
[5/8] Executing 'cuda_api_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Num Calls Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name
|
|
-------- --------------- --------- ---------- ---------- -------- -------- ----------- ---------------------------------
|
|
76.2 121358249 3 40452749.7 58133092.0 22181 63202976 35105542.2 cudaMemcpy
|
|
9.7 15479869 6 2579978.2 1263944.5 4198 8351576 3362372.0 cudaMalloc
|
|
6.3 10045470 1 10045470.0 10045470.0 10045470 10045470 0.0 cudaGetDeviceProperties_v2_v12000
|
|
3.2 5100454 12 425037.8 6347.5 180 1804402 614184.2 cudaFree
|
|
1.9 2948206 1 2948206.0 2948206.0 2948206 2948206 0.0 cudaMallocHost
|
|
1.6 2474178 1 2474178.0 2474178.0 2474178 2474178 0.0 cudaFreeHost
|
|
1.1 1675156 171 9796.2 2805.0 2675 1050069 80118.8 cudaLaunchKernel
|
|
0.1 154831 838 184.8 150.0 80 762 92.5 cuGetProcAddress_v2
|
|
0.0 39014 1 39014.0 39014.0 39014 39014 0.0 cudaMemset
|
|
0.0 17903 18 994.6 296.0 250 6583 1655.0 cudaEventCreateWithFlags
|
|
0.0 12222 4 3055.5 2619.5 982 6001 2113.3 cudaDeviceSynchronize
|
|
0.0 6803 18 377.9 251.0 210 2014 418.1 cudaEventDestroy
|
|
0.0 5841 4 1460.3 1362.5 1182 1934 328.0 cuInit
|
|
0.0 1352 3 450.7 120.0 90 1142 598.9 cuModuleGetLoadingMode
|
|
0.0 852 2 426.0 426.0 301 551 176.8 cudaGetDriverEntryPoint_v11030
|
|
|
|
[6/8] Executing 'cuda_gpu_kern_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Instances Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name
|
|
-------- --------------- --------- -------- -------- -------- -------- ----------- ----------------------------------------------------------------------------------------------------
|
|
88.5 45226826 49 922996.4 851682.0 243969 12757309 1750034.1 matmul_kernel(const float *, float *, const float *, const float *, int, int, int, int)
|
|
7.0 3578022 25 143120.9 143072.0 141120 146912 1197.9 layernorm_forward_kernel(float *, float *, float *, const float *, const float *, const float *, in…
|
|
2.3 1153027 12 96085.6 96112.5 95904 96192 94.9 preattention_scores_kernel(const float *, const float *, int, int, int, int, float *)
|
|
0.7 346208 12 28850.7 28688.0 28352 29760 446.4 attention_v_kernel(float *, const float *, const float *, int, int, int, int)
|
|
0.4 228480 12 19040.0 19040.0 18880 19232 79.6 gelu_forward_kernel(float *, const float *, int)
|
|
0.4 210304 12 17525.3 17520.0 17344 17792 116.8 softmax_forward_kernel(float *, float, const float *, int, int)
|
|
0.3 144992 24 6041.3 6048.0 5664 6464 367.5 residual_forward_kernel(float *, const float *, const float *, int)
|
|
0.2 84320 12 7026.7 7024.0 6880 7168 92.0 permute_kernel(float *, float *, float *, const float *, int, int, int, int)
|
|
0.2 80545 1 80545.0 80545.0 80545 80545 0.0 encoder_forward_kernel(float *, const int *, const float *, const float *, int, int, int)
|
|
0.1 70400 12 5866.7 5872.0 5792 5888 28.4 unpermute_kernel(const float *, float *, int, int, int, int)
|
|
|
|
[7/8] Executing 'cuda_gpu_mem_time_sum' stats report
|
|
|
|
Time (%) Total Time (ns) Count Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Operation
|
|
-------- --------------- ----- ---------- ---------- -------- -------- ----------- ----------------------------
|
|
88.9 63114351 2 31557175.5 31557175.5 832 63113519 44627409.0 [CUDA memcpy Host-to-Device]
|
|
10.5 7436977 1 7436977.0 7436977.0 7436977 7436977 0.0 [CUDA memcpy Device-to-Host]
|
|
0.6 407969 1 407969.0 407969.0 407969 407969 0.0 [CUDA memset]
|
|
|
|
[8/8] Executing 'cuda_gpu_mem_size_sum' stats report
|
|
|
|
Total (MB) Count Avg (MB) Med (MB) Min (MB) Max (MB) StdDev (MB) Operation
|
|
---------- ----- -------- -------- -------- -------- ----------- ----------------------------
|
|
497.905 2 248.952 248.952 0.001 497.904 352.070 [CUDA memcpy Host-to-Device]
|
|
232.392 1 232.392 232.392 232.392 232.392 0.000 [CUDA memset]
|
|
51.511 1 51.511 51.511 51.511 51.511 0.000 [CUDA memcpy Device-to-Host]
|
|
|
|
Generated:
|
|
/tmp/nsys-report-d644.nsys-rep
|
|
/tmp/nsys-report-9b06.sqlite
|