二维数组 cudaMallocPitch() 和三维数组 cudaMalloc3D() 的使用

▶ 使用函数 cudaMallocPitch() 和配套的函数 cudaMemcpy2D() 来使用二维数组。C 中二维数组内存分配是转化为一维数组，连贯紧凑，每次访问数组中的元素都必须从数组首元素开始遍历；而 cuda 中这样分配的二维数组内存保证了数组每一行首元素的地址值都按照 256 或 512 的倍数对齐，提高访问效率，但使得每行末尾元素与下一行首元素地址可能不连贯，使用指针寻址时要注意考虑尾部。

 // cuda_rumtime_api.h
 extern __host__ cudaError_t CUDARTAPI cudaMallocPitch(void **devPtr, size_t *pitch, size_t widthByte, size_t height);

 extern __host__ cudaError_t CUDARTAPI cudaMemcpy2D(void *dst, size_t dpitch, const void *src, size_t spitch, size_t width, size_t height, enum cudaMemcpyKind kind);

● cudaMAllocPitch() 传入存储器指针 **devPtr，偏移值的指针 *pitch，数组行字节数 widthByte，数组行数 height。函数返回后指针指向分配的内存（每行地址对齐到 AlignByte 字节，为 256B 或 512B），偏移值指针指向的值为该行实际字节数（= sizeof(datatype) * width + alignByte - 1) / alignByte）。

● cudaMemcpy2D() 传入目标存储器的指针 *dst，目标存储器行字节数 dpitch，源存储器指针 *src，源存储器行字节数 spitch，数组行字节数 width，数组行数 height，拷贝方向 kind。这里要求存储器行字节数不小于数组行字节数，多出来的部分就是每行尾部空白部分。

● 整个测试代码。

 #include <stdio.h>
 #include <malloc.h>
 #include <cuda_runtime_api.h>
 #include "device_launch_parameters.h"

 __global__ void myKernel(float* devPtr, int height, int width, int pitch)
 {
     int row, col;
     float *rowHead;

     for (row = ; row < height; row++)
     {
         rowHead = (float*)((char*)devPtr + row * pitch);

         for (col = ; col < width; col++)
         {
             printf("\t%f", rowHead[col]);// 逐个打印并自增 1
             rowHead[col]++;
         }
         printf("\n");
     }
 }

 int main()
 {
     size_t width = ;
     size_t height = ;
     float *h_data, *d_data;
     size_t pitch;

     h_data = (float *)malloc(sizeof(float)*width*height);
     for (int i = ; i < width*height; i++)
         h_data[i] = (float)i;

     printf("\n\tAlloc memory.");
     cudaMallocPitch((void **)&d_data, &pitch, sizeof(float)*width, height);
     printf("\n\tPitch = %d B\n", pitch);

     printf("\n\tCopy to Device.\n");
     cudaMemcpy2D(d_data, pitch, h_data, sizeof(float)*width, sizeof(float)*width, height, cudaMemcpyHostToDevice);

     myKernel << <,  >> > (d_data, height, width, pitch);
     cudaDeviceSynchronize();

     printf("\n\tCopy back to Host.\n");
     cudaMemcpy2D(h_data, sizeof(float)*width, d_data, pitch, sizeof(float)*width, height, cudaMemcpyDeviceToHost);

     for (int i = ; i < width*height; i++)
     {
         printf("\t%f", h_data[i]);
         if ((i + ) % width == )
             printf("\n");
     }               

     free(h_data);
     cudaFree(d_data);

     getchar();
     return ;
 }

● 输出结果：

        Alloc memory.
        Pitch =  B

        Copy to Device.
        0.000000        1.000000        2.000000        3.000000        4.000000        5.000000
        6.000000        7.000000        8.000000        9.000000        10.000000       11.000000
        12.000000       13.000000       14.000000       15.000000       16.000000       17.000000
        18.000000       19.000000       20.000000       21.000000       22.000000       23.000000
        24.000000       25.000000       26.000000       27.000000       28.000000       29.000000

        Copy back to Host.
        1.000000        2.000000        3.000000        4.000000        5.000000        6.000000
        7.000000        8.000000        9.000000        10.000000       11.000000       12.000000
        13.000000       14.000000       15.000000       16.000000       17.000000       18.000000
        19.000000       20.000000       21.000000       22.000000       23.000000       24.000000
        25.000000       26.000000       27.000000       28.000000       29.000000       30.000000

▶ 使用函数 cudaMalloc3D() 和配套的函数 cudaMemcpy3D() 来使用三维数组。因为涉及的参数较多，需要定义一些用来传参的结构，形式上和二维数组的使用有较大差距，不好看。

● 涉及的相关代码

 // driver_types.h
 struct cudaArray;                      // cuda 数组
 typedef struct cudaArray * cudaArray_t;// cuda 指针

 struct __device_builtin__ cudaPitchedPtr
 {
     void   *ptr;      // 实际数组指针（用完后要用 cudaFree() 释放掉）
     size_t  pitch;    // 数组行字节数
     size_t  xsize;    // 数组列数
     size_t  ysize;    // 数组行数
 };

 struct __device_builtin__ cudaExtent
 {
     size_t width;     // 数组行字节数
     size_t height;    // 数组行数
     size_t depth;     // 数组层数
 };

 struct __device_builtin__ cudaPos
 {
     size_t x;
     size_t y;
     size_t z;
 };

 struct __device_builtin__ cudaMemcpy3DParms
 {
     cudaArray_t            srcArray;  // 原数组指针
     struct cudaPos         srcPos;    // 原数组偏移
     struct cudaPitchedPtr  srcPtr;    // ？Pitched source memory address 

     cudaArray_t            dstArray;  // 目标数组指针
     struct cudaPos         dstPos;    // 目标数组偏移
     struct cudaPitchedPtr  dstPtr;    // ？Pitched destination memory address 

     struct cudaExtent      extent;    // 数组实际尺寸（去掉对齐用的空白部分）
     enum cudaMemcpyKind    kind;      // 拷贝类型
 };

 // driver_functions.h
 static __inline__ __host__ struct cudaPitchedPtr make_cudaPitchedPtr(void *d, size_t p, size_t xsz, size_t ysz)
 {                                     // 简单生成 cudaPitchedPtr 结构的方法
     struct cudaPitchedPtr s;

     s.ptr = d;
     s.pitch = p;
     s.xsize = xsz;
     s.ysize = ysz;

     return s;
 }

 static __inline__ __host__ struct cudaPos make_cudaPos(size_t x, size_t y, size_t z)
 {                                     // 简单的生成 cudaPos 结构的方法
     struct cudaPos p;

     p.x = x;
     p.y = y;
     p.z = z;

     return p;
 }

 static __inline__ __host__ struct cudaExtent make_cudaExtent(size_t w, size_t h, size_t d)
 {                                     // 简单的生成 cudaExtent 结构的方法
     struct cudaExtent e;

     e.width = w;
     e.height = h;
     e.depth = d;

     return e;
 }

 // cuda_runtime_api.h
 extern __host__ cudaError_t CUDARTAPI cudaMalloc3D(struct cudaPitchedPtr* pitchedDevPtr, struct cudaExtent extent);

 extern __host__ cudaError_t CUDARTAPI cudaMemcpy3D(const struct cudaMemcpy3DParms *p);

● 完整的测试程序

 #include <stdio.h>
 #include <malloc.h>
 #include <cuda_runtime_api.h>
 #include "device_launch_parameters.h"
 #include <driver_functions.h>

 __global__ void myKernel(cudaPitchedPtr devPitchedPtr, cudaExtent extent)
 {
     float * devPtr = (float *)devPitchedPtr.ptr;
     float *sliceHead, *rowHead;
         // 可以定义为 char * 作面、行迁移的时候直接加减字节数，取行内元素的时候再换回 float *

     for (int z = ; z < extent.depth; z++)
     {
         sliceHead = (float *)((char *)devPtr + z * devPitchedPtr.pitch * extent.height);
         for (int y = ; y < extent.height; y++)
         {
             rowHead = (float*)((char *)sliceHead + y * devPitchedPtr.pitch);
             for (int x = ; x < extent.width / sizeof(float); x++)// extent 存储的是行有效字节数，要除以元素大小
             {
                 printf("\t%f",rowHead[x]);// 逐个打印并自增 1
                 rowHead[x]++;
             }
             printf("\n");
         }
         printf("\n");
     }
 }

 int main()
 {
     size_t width = ;
     size_t height = ;
     size_t depth = ;
     float *h_data;

     cudaPitchedPtr d_data;
     cudaExtent extent;
     cudaMemcpy3DParms cpyParm;

     h_data = (float *)malloc(sizeof(float) * width * height * depth);
     for (int i = ; i < width * height * depth; i++)
         h_data[i] = (float)i;

     printf("\n\tAlloc memory.");
     extent = make_cudaExtent(sizeof(float) * width, height, depth);
     cudaMalloc3D(&d_data, extent);

     printf("\n\tCopy to Device.\n");
     cpyParm = {};
     cpyParm.srcPtr = make_cudaPitchedPtr((void*)h_data, sizeof(float) * width, width, height);
     cpyParm.dstPtr = d_data;
     cpyParm.extent = extent;
     cpyParm.kind = cudaMemcpyHostToDevice;
     cudaMemcpy3D(&cpyParm);

     myKernel << <,  >> > (d_data, extent);
     cudaDeviceSynchronize();

     printf("\n\tCopy back to Host.\n");
     cpyParm = {  };
     cpyParm.srcPtr = d_data;
     cpyParm.dstPtr = make_cudaPitchedPtr((void*)h_data, sizeof(float) * width, width, height);
     cpyParm.extent = extent;
     cpyParm.kind = cudaMemcpyDeviceToHost;
     cudaMemcpy3D(&cpyParm);

     for (int i = ; i < width*height*depth; i++)
     {
         printf("\t%f", h_data[i]);
         if ((i + ) % width == )
             printf("\n");
         if ((i + ) % (width*height) == )
             printf("\n");
     }               

     free(h_data);
     cudaFree(d_data.ptr);
     getchar();
     return ;
 }

● 输出结果：

        Alloc memory.
        Copy to Device.
        0.000000        1.000000
        2.000000        3.000000
        4.000000        5.000000

        6.000000        7.000000
        8.000000        9.000000
        10.000000       11.000000

        12.000000       13.000000
        14.000000       15.000000
        16.000000       17.000000

        18.000000       19.000000
        20.000000       21.000000
        22.000000       23.000000

        Copy back to Host.
        1.000000        2.000000
        3.000000        4.000000
        5.000000        6.000000

        7.000000        8.000000
        9.000000        10.000000
        11.000000       12.000000

        13.000000       14.000000
        15.000000       16.000000
        17.000000       18.000000

        19.000000       20.000000
        21.000000       22.000000
        23.000000       24.000000