pci p2p

概述

在2018年，针对pci支持p2pdma的驱动合入主线，没记错的话应该是4.20.

补丁如下：

commit 52916982af48d9f9fc01ad825259de1eb3a9b25e
Author: Logan Gunthorpe <logang@deltatee.com>
Date:   Thu Oct 4 15:27:35 2018 -0600

    PCI/P2PDMA: Support peer-to-peer memory

一、pci p2p DMA的条件

1.1内核配置

需要开启 CONFIG_PCI_P2PDMA 才能使用p2p的DMA。

需要设备具备内存，并以 ZONE_DEVICE 方式注册到内核中，暴露给驱动，需要注意的是，arm 默认不开启 CONFIG_ZONE_DEVICE，需要手动打开。

1.2 host bridge要求

从代码看，要求两个device 同属于同一个host_bridge,除此之外，还有一个白名单。

static bool host_bridge_whitelist(struct pci_dev *a, struct pci_dev *b,
				  bool warn)
{
	struct pci_host_bridge *host_a = pci_find_host_bridge(a->bus);
	struct pci_host_bridge *host_b = pci_find_host_bridge(b->bus);

	if (host_a == host_b)//caq:相同host brdige，没得说
		return __host_bridge_whitelist(host_a, true, warn);//caq:此时白名单检查传参为true

	if (__host_bridge_whitelist(host_a, false, warn) &&//caq:否则检查白名单
	    __host_bridge_whitelist(host_b, false, warn))//caq:此时传参为false
		return true;

	return false;
}
//caq:检查哪些host_bridge支持p2p
static bool __host_bridge_whitelist(struct pci_host_bridge *host,
				    bool same_host_bridge, bool warn)
{
	struct pci_dev *root = pci_host_bridge_dev(host);
	const struct pci_p2pdma_whitelist_entry *entry;
	unsigned short vendor, device;

	if (!root)
		return false;

	vendor = root->vendor;//caq:取vendorid
	device = root->device;//caq:取deviceid

	for (entry = pci_p2pdma_whitelist; entry->vendor; entry++) {
		if (vendor != entry->vendor || device != entry->device)
			continue;
		if (entry->flags & REQ_SAME_HOST_BRIDGE && !same_host_bridge)//caq:入参same_host_bridge 就是 true的话，则跳过
			return false;

		return true;//caq:如果 same_host_bridge 传参为true，则只要是在白名单之内的pci_host_bridge设备，都支持p2pdma
	}

	if (warn)//caq:如果没找到，则打印warning
		pci_warn(root, "Host bridge not in P2PDMA whitelist: %04x:%04x\n",
			 vendor, device);

	return false;
}

那么，是哪些设备在白名单之内呢？

static const struct pci_p2pdma_whitelist_entry {
	unsigned short vendor;
	unsigned short device;
	enum {
		REQ_SAME_HOST_BRIDGE	= 1 << 0,//caq:是否要求同一个host bridge
	} flags;
} pci_p2pdma_whitelist[] = {//caq:白名单列表
	/* Intel Xeon E5/Core i7 */
	{PCI_VENDOR_ID_INTEL,	0x3c00, REQ_SAME_HOST_BRIDGE},
	{PCI_VENDOR_ID_INTEL,	0x3c01, REQ_SAME_HOST_BRIDGE},
	/* Intel Xeon E7 v3/Xeon E5 v3/Core i7 */
	{PCI_VENDOR_ID_INTEL,	0x2f00, REQ_SAME_HOST_BRIDGE},
	{PCI_VENDOR_ID_INTEL,	0x2f01, REQ_SAME_HOST_BRIDGE},
	/* Intel SkyLake-E */
	{PCI_VENDOR_ID_INTEL,	0x2030, 0},//caq:不要求同一个host_bridge,也支持p2p
	{PCI_VENDOR_ID_INTEL,	0x2031, 0},
	{PCI_VENDOR_ID_INTEL,	0x2032, 0},
	{PCI_VENDOR_ID_INTEL,	0x2033, 0},
	{PCI_VENDOR_ID_INTEL,	0x2020, 0},
	{PCI_VENDOR_ID_INTEL,	0x09a2, 0},
	{}
};

简单这么一看，只有intel的某些host_bridge支持p2pdma，那么，amd能忍么？

1.3 cpu要求

前面说到intel 通过白名单添加host_bridge的方式来支持跨host_bridge的p2p dma，amd一看，他也要有，然后提交了如下patch：

commit 7d5b10fcb81e511ddf79c1c6b7f6efb282f80680
Author: Alex Deucher <alexdeucher@gmail.com>
Date:   Mon Apr 6 15:42:01 2020 -0400

    PCI/P2PDMA: Add AMD Zen Raven and Renoir Root Ports to whitelist

    According to the hardware architect, pre-Zen parts support p2p writes and
    Zen parts support both p2p reads and writes.

    Add entries for Zen parts Raven (0x15d0) and Renoir (0x1630).

    Link: https://lore.kernel.org/r/20200406194201.846411-1-alexander.deucher@amd.com
    Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
    Signed-off-by: Bjorn Helgaas <bhelgaas@google.com>
    Acked-by: Christian König <christian.koenig@amd.com>
    Acked-by: Huang Rui <ray.huang@amd.com>

diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c
index b73b10bce0df..e8e444eeb1cd 100644
--- a/drivers/pci/p2pdma.c
+++ b/drivers/pci/p2pdma.c
@@ -282,6 +282,8 @@ static const struct pci_p2pdma_whitelist_entry {
 } pci_p2pdma_whitelist[] = {
        /* AMD ZEN */
        {PCI_VENDOR_ID_AMD,     0x1450, 0},
+       {PCI_VENDOR_ID_AMD,     0x15d0, 0},
+       {PCI_VENDOR_ID_AMD,     0x1630, 0},

        /* Intel Xeon E5/Core i7 */
        {PCI_VENDOR_ID_INTEL,   0x3c00, REQ_SAME_HOST_BRIDGE},

当前在此之前还添加了一个0x1450，但是加着加着amd缓过头来了，amd的zen架构之后，都支持跨host_bridge 来支持p2pdma，那岂不是白名单得一直加下去？

所以，amd干脆另起炉灶：

commit dea286bb71baded7d2fb4f090e3b9fd2c1ccac58
Author: Logan Gunthorpe <logang@deltatee.com>
Date:   Wed Jul 29 17:18:44 2020 -0600

    PCI/P2PDMA: Allow P2PDMA on AMD Zen and newer CPUs

    Allow P2PDMA if the CPU vendor is AMD and family is 0x17 (Zen) or greater.

    [bhelgaas: commit log, simplify #if/#else/#endif]
    Link: https://lore.kernel.org/r/20200729231844.4653-1-logang@deltatee.com
    Signed-off-by: Logan Gunthorpe <logang@deltatee.com>
    Signed-off-by: Bjorn Helgaas <bhelgaas@google.com>
    Reviewed-by: Alex Deucher <alexander.deucher@amd.com>
    Cc: Christian König <christian.koenig@amd.com>
    Cc: Huang Rui <ray.huang@amd.com>

diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c
index e8e444eeb1cd..1ec61fced4c3 100644
--- a/drivers/pci/p2pdma.c
+++ b/drivers/pci/p2pdma.c
@@ -273,6 +273,19 @@ static void seq_buf_print_bus_devfn(struct seq_buf *buf, struct pci_dev *pdev)
        seq_buf_printf(buf, "%s;", pci_name(pdev));
 }

+static bool cpu_supports_p2pdma(void)
+{
+#ifdef CONFIG_X86
+       struct cpuinfo_x86 *c = &cpu_data(0);
+
+       /* Any AMD CPU whose family ID is Zen or newer supports p2pdma */
+       if (c->x86_vendor == X86_VENDOR_AMD && c->x86 >= 0x17)//caq:amd另起炉灶，只要是zen或者更新的架构，都支持，但比如zen被授权出去，host_bridge被第三方修改，能保证么？
+               return true;
+#endif
+
+       return false;
+}
+
 static const struct pci_p2pdma_whitelist_entry {
        unsigned short vendor;
        unsigned short device;
@@ -280,11 +293,6 @@ static const struct pci_p2pdma_whitelist_entry {
                REQ_SAME_HOST_BRIDGE    = 1 << 0,
        } flags;
 } pci_p2pdma_whitelist[] = {
-       /* AMD ZEN */
-       {PCI_VENDOR_ID_AMD,     0x1450, 0},
-       {PCI_VENDOR_ID_AMD,     0x15d0, 0},
-       {PCI_VENDOR_ID_AMD,     0x1630, 0},
-
        /* Intel Xeon E5/Core i7 */
        {PCI_VENDOR_ID_INTEL,   0x3c00, REQ_SAME_HOST_BRIDGE},
        {PCI_VENDOR_ID_INTEL,   0x3c01, REQ_SAME_HOST_BRIDGE},
@@ -473,7 +481,8 @@ upstream_bridge_distance(struct pci_dev *provider, struct pci_dev *client,
                                              acs_redirects, acs_list);

        if (map_type == PCI_P2PDMA_MAP_THRU_HOST_BRIDGE) {
-               if (!host_bridge_whitelist(provider, client))
+               if (!cpu_supports_p2pdma() &&
+                   !host_bridge_whitelist(provider, client))
                        map_type = PCI_P2PDMA_MAP_NOT_SUPPORTED;
        }

总而言之，就是zen系列之后的，全支持，不再跟intel那样加白名单了，有点霸气，但是不是还会改回来还未知，比如amd做了一款编号大于 0x17 的cpu，但是又不支持跨host_bridge的p2pdma咋办？