OpenACC 书上的范例代码（Jacobi 迭代），part 2

▶ 使用Jacobi 迭代求泊松方程的数值解

● 首次使用 OpenACC 进行加速。

■ win10下使用 -acc 选项时报错找不到结构 _timb，我把其定义（实际是结构 __timeb64）写到本文件中，发现计时器只能精确到秒，Ubuntu 中正常运行。

■ 把 win10 把计时器改回 <time.h> 中的 clock_t（下一部分代码起都改了），可以精确计时。

■ win10 下使用 PGI_ACC_TIME=1 来计时会报错：Error: internal error: invalid thread id，暂时无解。

 #include <stdio.h>
 #include <stdlib.h>
 #include <math.h>
 #include <openacc.h>

 #if defined(_WIN32) || defined(_WIN64)
 #include <C:\Program Files\PGI\win64\19.4\include\wrap\sys\timeb.h>
 #define gettime(a)  _ftime(a)
 #define usec(t1,t2) ((((t2).time - (t1).time) * 1000 + (t2).millitm - (t1).millitm))
 //typedef struct _timeb timestruct;         // 使用 -acc 选项时报错找不到 _timb 结构，只能把原始定义写在这里
 typedef struct
 {
     __time64_t     time;
     unsigned short millitm;
     short          timezone;
     short          dstflag;
 } timestruct;
 #else
 #include <sys/time.h>
 #define gettime(a)  gettimeofday(a, NULL)
 #define usec(t1,t2) (((t2).tv_sec - (t1).tv_sec) * 1000000 + (t2).tv_usec - (t1).tv_usec)
 typedef struct timeval timestruct;
 #endif

 inline float uval(float x, float y)
 {
     return x * x + y * y;
 }

 int main()
 {
     const int row = , col = ;
     const float height = 1.0, width = 2.0;
     const float hx = height / row, wy = width / col;
     const float fij = -4.0f;
     const float hx2 = hx * hx, wy2 = wy * wy, c1 = hx2 * wy2, c2 = 1.0f / (2.0 * (hx2 + wy2));
     const int maxIter = ;
     const int colPlus = col + ;                        

     float *restrict u0 = (float *)malloc(sizeof(float)*(row + )*colPlus);
     float *restrict u1 = (float *)malloc(sizeof(float)*(row + )*colPlus);
     float *utemp = NULL;                                                

     // 初始化
     for (int ix = ; ix <= row; ix++)
     {
         u0[ix*colPlus + ] = u1[ix*colPlus + ] = uval(ix * hx, 0.0f);
         u0[ix*colPlus + col] = u1[ix*colPlus + col] = uval(ix*hx, col * wy);
     }
     for (int jy = ; jy <= col; jy++)
     {
         u0[jy] = u1[jy] = uval(0.0f, jy * wy);
         u0[row*colPlus + jy] = u1[row*colPlus + jy] = uval(row*hx, jy * wy);
     }
     for (int ix = ; ix < row; ix++)
     {
         for (int jy = ; jy < col; jy++)
             u0[ix*colPlus + jy] = 0.0f;
     }

     // 计算
     timestruct t1, t2;
     gettime(&t1);
     acc_init(acc_device_nvidia);                        // 初始化设备，定义在 openacc.h 中，acc_device_nvidia 可用 4 代替
     for (int iter = ; iter < maxIter; iter++)
     {
 #pragma acc kernels copy(u0[0:(row + 1) * colPlus], u1[0:(row + 1) * colPlus])  // 使用 kernels 和 loop
         #pragma acc loop independent                    // 显式说明各次循环之间独立
         for (int ix = ; ix < row; ix++)
         {
             #pragma acc loop independent                // PGI 18.04 上不加这一行也行，PGI 19.04 上不加这一行会报
             for (int jy = ; jy < col; jy++)            // Complex loop carried dependence of u0->,u1-> prevents parallelization. Inner sequential loop scheduled on accelerator.
             {
                 u1[ix*colPlus + jy] = (c1*fij + wy2 * (u0[(ix - )*colPlus + jy] + u0[(ix + )*colPlus + jy]) + \
                     hx2 * (u0[ix*colPlus + jy - ] + u0[ix*colPlus + jy + ])) * c2;
             }
         }
         utemp = u0, u0 = u1, u1 = utemp;
     }
     gettime(&t2);    

     long long timeElapse = usec(t1, t2);
 #if defined(_WIN32) || defined(_WIN64)
     printf("\nElapsed time: %13ld ms.\n", timeElapse);
 #else
     printf("\nElapsed time: %13ld us.\n", timeElapse);
 #endif
     free(u0);
     free(u1);
     acc_shutdown(acc_device_nvidia);                    // 关闭设备
     //getchar();
     return ;
 }

● 输出结果，win10 下改了计时器以后运行时间 2370 ms

cuan@CUAN:~$ pgcc -c99 -acc -Minfo main.c -o main.exe
main:
     , Memory zero idiom, loop replaced by call to __c_mzero4
     , Generating copy(u1[:colPlus*(row+)],u0[:colPlus*(row+)])
         FMA (fused multiply-add) instruction(s) generated
     , Loop is parallelizable
     , Loop is parallelizable
         Generating Tesla code
         , #pragma acc loop gang, vector(4) /* blockIdx.y threadIdx.y */
         , #pragma acc loop gang, vector(32) /* blockIdx.x threadIdx.x */
uval:
     , FMA (fused multiply-add) instruction(s) generated
cuan@CUAN:~$ ./main.exe
launch CUDA kernel  file=/home/cuan/main.c function=main line= device= threadid= num_gangs= num_workers= vector_length= grid=32x1024 block=32x4

... //98 行相同的内容，每个展开的循环一个 kernel

launch CUDA kernel  file=/home/cuan/main.c function=main line= device= threadid= num_gangs= num_workers= vector_length= grid=32x1024 block=32x4

Elapsed time:        us.

Accelerator Kernel Timing data
/home/cuan/main.c
  main  NVIDIA  devicenum=
    time(us): ,,
    : data region reached  times
        : data copyin transfers:
             device time(us): total=, max=, min=, avg=,
        : data copyout transfers:
             device time(us): total=, max=, min= avg=
    : compute region reached  times
        : kernel launched  times
            grid: [32x1024]  block: [32x4]
             device time(us): total=, max= min= avg=
            elapsed time(us): total=, max= min= avg=

cuan@CUAN:~$ pgcc -c99 -acc -Minfo main.c -o main-fast.exe -fast    // 使用 -fast 选项
main:
     , uval inlined, size= (inline) file main.c ()             // 47 ～ 53 行也尝试优化
          , Loop not fused: different loop trip count
              Loop not vectorized: data dependency
              Loop unrolled  times
     , uval inlined, size= (inline) file main.c ()
     , uval inlined, size= (inline) file main.c ()
          , Loop not vectorized: data dependency
              Loop not vectorized: may not be beneficial
              Loop unrolled  times
     , uval inlined, size= (inline) file main.c ()
     , Memory zero idiom, loop replaced by call to __c_mzero4
     , Generating copy(u1[:colPlus*(row+)],u0[:colPlus*(row+)])
         Loop not vectorized/parallelized: contains call            // 65 行拒绝优化
         FMA (fused multiply-add) instruction(s) generated
     , Loop is parallelizable
         Loop not fused: no successor loop                          // 69 行拒绝优化
     , Loop is parallelizable
         Generating Tesla code
         , #pragma acc loop gang, vector(4) /* blockIdx.y threadIdx.y */
         , #pragma acc loop gang, vector(32) /* blockIdx.x threadIdx.x */
     , Loop not vectorized: data dependency                       // 72 行拒绝优化
         Loop unrolled  times
cuan@CUAN:~$ ./main-fast.exe
launch CUDA kernel  file=/home/cuan/main.c function=main line= device= threadid= num_gangs= num_workers= vector_length= grid=32x1024 block=32x4

...                                                                 // 跟没有 -fast 一样

launch CUDA kernel  file=/home/cuan/main.c function=main line= device= threadid= num_gangs= num_workers= vector_length= grid=32x1024 block=32x4

Elapsed time:        us.                                      // 速度影响不大

Accelerator Kernel Timing data
/home/cuan/main.c
  main  NVIDIA  devicenum=
    time(us): ,,
    : data region reached  times
        : data copyin transfers:
             device time(us): total=, max=, min=, avg=,
        : data copyout transfers:
             device time(us): total=, max=, min= avg=
    : compute region reached  times
        : kernel launched  times
            grid: [32x1024]  block: [32x4]
             device time(us): total=, max= min= avg=
            elapsed time(us): total=, max=, min= avg=

● 使用 Nvvp 分析，发现密密麻麻的都是内存拷贝，计算反而是瞬间完成：

■ 注意数据管理导语 copy 是在 for(int iter=1;...) 内部的，所以会生成 100 个独立内核来运行，如果把它放到外侧（第 67 行改到 65 行之前）则会报错有以下报错：

PGC-S--Cannot determine bounds for array u0 (main.c: )        // 指针中不含有数组大小的信息
PGC-S--Cannot determine bounds for array u1 (main.c: )
main:
     , Memory zero idiom, loop replaced by call to __c_mzero4
     , Loop carried scalar dependence for u0 at line ,
         Scalar last value needed after loop for u0 at line
         Loop carried scalar dependence for u1 at line
         Scalar last value needed after loop for u1 at line
         Scalar last value needed after loop for u0 at line
         Parallelization would require privatization of array u1[(colPlus)*i2+colPlus+:i1+i2+]
         Generating Tesla code
         , #pragma acc loop seq                                   // 串行执行
         , #pragma acc loop seq
         , #pragma acc loop vector(128) /* threadIdx.x */
     , Loop carried scalar dependence for u1 at line
         Loop carried scalar dependence for u0 at line
         Loop carried scalar dependence for u1 at line
         Parallelization would require privatization of array u1[(colPlus)*i2+colPlus+:i1+i2+]

● 使用静态数组，此时可将数据管理导语 copy 放到 for之前，win10 下能运行，Linux上默认栈为 8 MB（ulimited -a），最大可调整为 64 MB（ulimited -s），两个数组除非调小否则刚好装不下，会报 dump core。

 #include <stdio.h>
 #include <stdlib.h>
 #include <math.h>
 #include <time.h>
 #include <openacc.h>

 #if defined(_WIN32) || defined(_WIN64)
     #include <C:\Program Files\PGI\win64\19.4\include\wrap\sys\timeb.h>
     #define timestruct clock_t
     #define gettime(a) (*(a) = clock())
     #define usec(t1,t2) (t2 - t1)
 #else
     #include <sys/time.h>
     #define gettime(a)  gettimeofday(a, NULL)
     #define usec(t1,t2) (((t2).tv_sec - (t1).tv_sec) * 1000000 + (t2).tv_usec - (t1).tv_usec)
     typedef struct timeval timestruct;
 #endif

 inline float uval(float x, float y)
 {
     return x * x + y * y;
 }

 int main()
 {
     const int row = , col = , colPlus = col + ;
     const float height = 1.0, width = 2.0;
     const float hx = height / row, wy = width / col;
     const float fij = -4.0f;
     const float hx2 = hx * hx, wy2 = wy * wy, c1 = hx2 * wy2, c2 = 1.0f / (2.0 * (hx2 + wy2));
     const int maxIter = ;                            

     float u0[(row + ) * (col + )];
     float u1[(row + ) * (col + )];

     // 初始化
     for (int ix = ; ix < (row+) * colPlus; ix++)
         u0[ix] = 0.0f;
     for (int ix = ; ix <= row; ix++)
     {
         u0[ix*colPlus + ] = u1[ix*colPlus + ] = uval(ix * hx, 0.0f);
         u0[ix*colPlus + col] = u1[ix*colPlus + col] = uval(ix*hx, col * wy);
     }
     for (int jy = ; jy <= col; jy++)
     {
         u0[jy] = u1[jy] = uval(0.0f, jy * wy);
         u0[row*colPlus + jy] = u1[row*colPlus + jy] = uval(row*hx, jy * wy);
     }    

     // 计算
     timestruct t1, t2;
     gettime(&t1);
     acc_init(acc_device_nvidia);
 #pragma acc kernels copy(u0[0:(row + 1) * colPlus], u1[0:(row + 1) * colPlus])  // 换到 for (int iter = 0;...) 前面
     for (int iter = ; iter < maxIter; iter++)
     {
         if (iter % )                                                           // iter 奇偶性分类源数组和目标数组
         {
 #pragma acc loop independent
             for (int ix = ; ix < row; ix++)
             {
                 for (int jy = ; jy < col; jy++)
                 {
                     u0[ix*colPlus + jy] = (wy2 * (u1[(ix - ) * colPlus + jy] + u1[(ix + ) * colPlus + jy]) + \
                         hx2 * (u1[ix * colPlus + (jy - )] + u1[ix * colPlus + (jy + )]) + c1 * fij) * c2;
                 }
             }
         }
         else
         {
 #pragma acc loop independent
             for (int ix = ; ix < row; ix++)
             {
                 for (int jy = ; jy < col; jy++)
                 {
                     u1[ix*colPlus + jy] = (wy2 * (u0[(ix - ) * colPlus + jy] + u0[(ix + ) * colPlus + jy]) + \
                         hx2 * (u0[ix * colPlus + (jy - )] + u0[ix * colPlus + (jy + )]) + c1 * fij) * c2;
                 }
             }
         }
     }
     gettime(&t2);    

     long long timeElapse = usec(t1, t2);
 #if defined(_WIN32) || defined(_WIN64)
     printf("\nElapsed time: %13ld ms.\n", timeElapse);
 #else
     printf("\nElapsed time: %13ld us.\n", timeElapse);
 #endif
     acc_shutdown(acc_device_nvidia);
     //getchar();
     return ;
 }

● 输出结果，在 Windows 里结果如下，在 WSL 里运行时间为 849 ms● 使用 Nvvp 分析，发现内存拷贝只有首尾两次，中间全在 GPU 上计算：