diff options
| -rw-r--r-- | Documentation/driver-api/cxl/linux/dax-driver.rst | 4 | ||||
| -rw-r--r-- | drivers/cxl/core/core.h | 2 | ||||
| -rw-r--r-- | drivers/cxl/core/mbox.c | 65 | ||||
| -rw-r--r-- | drivers/cxl/core/mce.c | 27 | ||||
| -rw-r--r-- | drivers/cxl/core/memdev.c | 67 | ||||
| -rw-r--r-- | drivers/cxl/core/pci.c | 1 | ||||
| -rw-r--r-- | drivers/cxl/core/port.c | 1 | ||||
| -rw-r--r-- | drivers/cxl/core/region.c | 43 | ||||
| -rw-r--r-- | drivers/cxl/core/regs.c | 1 | ||||
| -rw-r--r-- | drivers/cxl/cxl.h | 8 | ||||
| -rw-r--r-- | drivers/cxl/cxlmem.h | 4 | ||||
| -rw-r--r-- | drivers/cxl/cxlpci.h | 12 | ||||
| -rw-r--r-- | drivers/cxl/pci.c | 7 | ||||
| -rw-r--r-- | include/cxl/cxl.h | 2 | ||||
| -rw-r--r-- | include/cxl/pci.h | 22 |
15 files changed, 132 insertions, 134 deletions
diff --git a/Documentation/driver-api/cxl/linux/dax-driver.rst b/Documentation/driver-api/cxl/linux/dax-driver.rst index 10d953a2167b..72c85a0f8606 100644 --- a/Documentation/driver-api/cxl/linux/dax-driver.rst +++ b/Documentation/driver-api/cxl/linux/dax-driver.rst @@ -35,9 +35,9 @@ will be exposed to the kernel page allocator in the user-selected memory zone. The :code:`memmap_on_memory` setting (both global and DAX device local) -dictates where the kernell will allocate the :code:`struct folio` descriptors +dictates where the kernel will allocate the :code:`struct folio` descriptors for this memory will come from. If :code:`memmap_on_memory` is set, memory hotplug will set aside a portion of the memory block capacity to allocate folios. If unset, the memory is allocated via a normal :code:`GFP_KERNEL` -allocation - and as a result will most likely land on the local NUM node of the +allocation - and as a result will most likely land on the local NUMA node of the CPU executing the hotplug operation. diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h index 07555ae63859..f7cebb026552 100644 --- a/drivers/cxl/core/core.h +++ b/drivers/cxl/core/core.h @@ -101,6 +101,8 @@ void __iomem *devm_cxl_iomap_block(struct device *dev, resource_size_t addr, struct dentry *cxl_debugfs_create_dir(const char *dir); int cxl_dpa_set_part(struct cxl_endpoint_decoder *cxled, enum cxl_partition_mode mode); +struct cxl_memdev_state; +int cxl_mem_get_partition_info(struct cxl_memdev_state *mds); int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, u64 size); int cxl_dpa_free(struct cxl_endpoint_decoder *cxled); resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled); diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index 7c6c5b7450a5..79543103ac29 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -11,7 +11,6 @@ #include "core.h" #include "trace.h" -#include "mce.h" static bool cxl_raw_allow_all; @@ -380,11 +379,7 @@ static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox_cmd, } } - /* Prepare to handle a full payload for variable sized output */ - if (out_size == CXL_VARIABLE_PAYLOAD) - mbox_cmd->size_out = cxl_mbox->payload_size; - else - mbox_cmd->size_out = out_size; + mbox_cmd->size_out = min_t(size_t, out_size, cxl_mbox->payload_size); if (mbox_cmd->size_out) { mbox_cmd->payload_out = kvzalloc(mbox_cmd->size_out, GFP_KERNEL); @@ -1152,7 +1147,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_mem_get_event_records, "CXL"); * * See CXL @8.2.9.5.2.1 Get Partition Info */ -static int cxl_mem_get_partition_info(struct cxl_memdev_state *mds) +int cxl_mem_get_partition_info(struct cxl_memdev_state *mds) { struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; struct cxl_mbox_get_partition_info pi; @@ -1308,55 +1303,6 @@ int cxl_mem_sanitize(struct cxl_memdev *cxlmd, u16 cmd) return -EBUSY; } -static void add_part(struct cxl_dpa_info *info, u64 start, u64 size, enum cxl_partition_mode mode) -{ - int i = info->nr_partitions; - - if (size == 0) - return; - - info->part[i].range = (struct range) { - .start = start, - .end = start + size - 1, - }; - info->part[i].mode = mode; - info->nr_partitions++; -} - -int cxl_mem_dpa_fetch(struct cxl_memdev_state *mds, struct cxl_dpa_info *info) -{ - struct cxl_dev_state *cxlds = &mds->cxlds; - struct device *dev = cxlds->dev; - int rc; - - if (!cxlds->media_ready) { - info->size = 0; - return 0; - } - - info->size = mds->total_bytes; - - if (mds->partition_align_bytes == 0) { - add_part(info, 0, mds->volatile_only_bytes, CXL_PARTMODE_RAM); - add_part(info, mds->volatile_only_bytes, - mds->persistent_only_bytes, CXL_PARTMODE_PMEM); - return 0; - } - - rc = cxl_mem_get_partition_info(mds); - if (rc) { - dev_err(dev, "Failed to query partition information\n"); - return rc; - } - - add_part(info, 0, mds->active_volatile_bytes, CXL_PARTMODE_RAM); - add_part(info, mds->active_volatile_bytes, mds->active_persistent_bytes, - CXL_PARTMODE_PMEM); - - return 0; -} -EXPORT_SYMBOL_NS_GPL(cxl_mem_dpa_fetch, "CXL"); - int cxl_get_dirty_count(struct cxl_memdev_state *mds, u32 *count) { struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; @@ -1526,7 +1472,6 @@ struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev, u64 serial, u16 dvsec) { struct cxl_memdev_state *mds; - int rc; mds = devm_cxl_dev_state_create(dev, CXL_DEVTYPE_CLASSMEM, serial, dvsec, struct cxl_memdev_state, cxlds, @@ -1538,12 +1483,6 @@ struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev, u64 serial, mutex_init(&mds->event.log_lock); - rc = devm_cxl_register_mce_notifier(dev, &mds->mce_notifier); - if (rc == -EOPNOTSUPP) - dev_warn(dev, "CXL MCE unsupported\n"); - else if (rc) - return ERR_PTR(rc); - return mds; } EXPORT_SYMBOL_NS_GPL(cxl_memdev_state_create, "CXL"); diff --git a/drivers/cxl/core/mce.c b/drivers/cxl/core/mce.c index ff8d078c6ca1..65fed913b221 100644 --- a/drivers/cxl/core/mce.c +++ b/drivers/cxl/core/mce.c @@ -4,16 +4,16 @@ #include <linux/notifier.h> #include <linux/set_memory.h> #include <asm/mce.h> -#include <cxlmem.h> +#include <cxl.h> +#include "core.h" #include "mce.h" static int cxl_handle_mce(struct notifier_block *nb, unsigned long val, void *data) { - struct cxl_memdev_state *mds = container_of(nb, struct cxl_memdev_state, - mce_notifier); - struct cxl_memdev *cxlmd = mds->cxlds.cxlmd; - struct cxl_port *endpoint = cxlmd->endpoint; + struct cxl_region *cxlr = container_of(nb, struct cxl_region, + mce_notifier); + struct cxl_region_params *p = &cxlr->params; struct mce *mce = data; u64 spa, spa_alias; unsigned long pfn; @@ -21,26 +21,25 @@ static int cxl_handle_mce(struct notifier_block *nb, unsigned long val, if (!mce || !mce_usable_address(mce)) return NOTIFY_DONE; - if (!endpoint) - return NOTIFY_DONE; - spa = mce->addr & MCI_ADDR_PHYSADDR; - pfn = spa >> PAGE_SHIFT; - if (!pfn_valid(pfn)) + if (!cxl_resource_contains_addr(p->res, spa)) return NOTIFY_DONE; - spa_alias = cxl_port_get_spa_cache_alias(endpoint, spa); - if (spa_alias == ~0ULL) - return NOTIFY_DONE; + if (spa >= p->res->start + p->cache_size) + spa_alias = spa - p->cache_size; + else + spa_alias = spa + p->cache_size; pfn = spa_alias >> PAGE_SHIFT; + if (!pfn_valid(pfn)) + return NOTIFY_DONE; /* * Take down the aliased memory page. The original memory page flagged * by the MCE will be taken cared of by the standard MCE handler. */ - dev_emerg(mds->cxlds.dev, "Offlining aliased SPA address0: %#llx\n", + dev_emerg(&cxlr->dev, "Offlining aliased SPA address0: %#llx\n", spa_alias); if (!memory_failure(pfn, 0)) set_mce_nospec(pfn); diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c index 33a3d2e7b13a..2e457b1ebc7d 100644 --- a/drivers/cxl/core/memdev.c +++ b/drivers/cxl/core/memdev.c @@ -594,6 +594,73 @@ bool is_cxl_memdev(const struct device *dev) } EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, "CXL"); +static void add_part(struct cxl_dpa_info *info, u64 start, u64 size, enum cxl_partition_mode mode) +{ + int i = info->nr_partitions; + + if (size == 0) + return; + + info->part[i].range = (struct range) { + .start = start, + .end = start + size - 1, + }; + info->part[i].mode = mode; + info->nr_partitions++; +} + +int cxl_mem_dpa_fetch(struct cxl_memdev_state *mds, struct cxl_dpa_info *info) +{ + struct cxl_dev_state *cxlds = &mds->cxlds; + struct device *dev = cxlds->dev; + int rc; + + if (!cxlds->media_ready) { + info->size = 0; + return 0; + } + + info->size = mds->total_bytes; + + if (mds->partition_align_bytes == 0) { + add_part(info, 0, mds->volatile_only_bytes, CXL_PARTMODE_RAM); + add_part(info, mds->volatile_only_bytes, + mds->persistent_only_bytes, CXL_PARTMODE_PMEM); + return 0; + } + + rc = cxl_mem_get_partition_info(mds); + if (rc) { + dev_err(dev, "Failed to query partition information\n"); + return rc; + } + + add_part(info, 0, mds->active_volatile_bytes, CXL_PARTMODE_RAM); + add_part(info, mds->active_volatile_bytes, mds->active_persistent_bytes, + CXL_PARTMODE_PMEM); + + return 0; +} +EXPORT_SYMBOL_NS_GPL(cxl_mem_dpa_fetch, "CXL"); + + +/** + * cxl_set_capacity: initialize dpa by a driver without a mailbox. + * + * @cxlds: pointer to cxl_dev_state + * @capacity: device volatile memory size + */ +int cxl_set_capacity(struct cxl_dev_state *cxlds, u64 capacity) +{ + struct cxl_dpa_info range_info = { + .size = capacity, + }; + + add_part(&range_info, 0, capacity, CXL_PARTMODE_RAM); + return cxl_dpa_setup(cxlds, &range_info); +} +EXPORT_SYMBOL_NS_GPL(cxl_set_capacity, "CXL"); + /** * set_exclusive_cxl_commands() - atomically disable user cxl commands * @mds: The device state to operate on diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index e4338fd7e01b..9d807c1a002c 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -6,6 +6,7 @@ #include <linux/delay.h> #include <linux/pci.h> #include <linux/pci-doe.h> +#include <cxl/pci.h> #include <linux/aer.h> #include <cxlpci.h> #include <cxlmem.h> diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 1215ee4f4035..cb633e19151b 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -11,6 +11,7 @@ #include <linux/idr.h> #include <linux/node.h> #include <cxl/einj.h> +#include <cxl/pci.h> #include <cxlmem.h> #include <cxlpci.h> #include <cxl.h> diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index 1e211542b6b6..578622240401 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -15,6 +15,7 @@ #include <cxlmem.h> #include <cxl.h> #include "core.h" +#include "mce.h" /** * DOC: cxl core region @@ -3859,34 +3860,6 @@ int cxl_add_to_region(struct cxl_endpoint_decoder *cxled) } EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, "CXL"); -u64 cxl_port_get_spa_cache_alias(struct cxl_port *endpoint, u64 spa) -{ - struct cxl_region_ref *iter; - unsigned long index; - - if (!endpoint) - return ~0ULL; - - guard(rwsem_write)(&cxl_rwsem.region); - - xa_for_each(&endpoint->regions, index, iter) { - struct cxl_region_params *p = &iter->region->params; - - if (cxl_resource_contains_addr(p->res, spa)) { - if (!p->cache_size) - return ~0ULL; - - if (spa >= p->res->start + p->cache_size) - return spa - p->cache_size; - - return spa + p->cache_size; - } - } - - return ~0ULL; -} -EXPORT_SYMBOL_NS_GPL(cxl_port_get_spa_cache_alias, "CXL"); - static int is_system_ram(struct resource *res, void *arg) { struct cxl_region *cxlr = arg; @@ -4217,6 +4190,20 @@ static int cxl_region_probe(struct device *dev) if (rc) return rc; + /* + * Regions fronted by an extended linear cache need the MCE notifier to + * offline the aliased page on a memory error. + */ + if (p->cache_size) { + rc = devm_cxl_register_mce_notifier(&cxlr->dev, + &cxlr->mce_notifier); + if (rc == -EOPNOTSUPP) + dev_warn(&cxlr->dev, + "CONFIG_CXL_MCE disabled, MCE notifier not registered\n"); + else if (rc) + return rc; + } + rc = cxl_region_setup_poison(cxlr); if (rc) return rc; diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c index 93710cf4f0a6..20c2d9fbcfe7 100644 --- a/drivers/cxl/core/regs.c +++ b/drivers/cxl/core/regs.c @@ -4,6 +4,7 @@ #include <linux/device.h> #include <linux/slab.h> #include <linux/pci.h> +#include <cxl/pci.h> #include <cxlmem.h> #include <cxlpci.h> #include <pmu.h> diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index c0e5308e4d1b..bdd0132c999d 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -478,6 +478,7 @@ struct cxl_region_params { * @coord: QoS access coordinates for the region * @node_notifier: notifier for setting the access coordinates to node * @adist_notifier: notifier for calculating the abstract distance of node + * @mce_notifier: notifier for MCE */ struct cxl_region { struct device dev; @@ -493,6 +494,7 @@ struct cxl_region { struct access_coordinate coord[ACCESS_COORDINATE_MAX]; struct notifier_block node_notifier; struct notifier_block adist_notifier; + struct notifier_block mce_notifier; }; struct cxl_nvdimm_bridge { @@ -870,7 +872,6 @@ bool is_cxl_pmem_region(struct device *dev); struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev); int cxl_add_to_region(struct cxl_endpoint_decoder *cxled); struct cxl_dax_region *to_cxl_dax_region(struct device *dev); -u64 cxl_port_get_spa_cache_alias(struct cxl_port *endpoint, u64 spa); bool cxl_region_contains_resource(const struct resource *res); #else static inline bool is_cxl_pmem_region(struct device *dev) @@ -889,11 +890,6 @@ static inline struct cxl_dax_region *to_cxl_dax_region(struct device *dev) { return NULL; } -static inline u64 cxl_port_get_spa_cache_alias(struct cxl_port *endpoint, - u64 spa) -{ - return 0; -} static inline bool cxl_region_contains_resource(const struct resource *res) { return false; diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h index ed419d0c59f2..c401e3a1af06 100644 --- a/drivers/cxl/cxlmem.h +++ b/drivers/cxl/cxlmem.h @@ -183,7 +183,7 @@ static inline struct cxl_ep *cxl_ep_load(struct cxl_port *port, C(MBUNSUPPORTED, -ENXIO, "unsupported on the mailbox it was issued on"),\ C(PAYLOADLEN, -ENXIO, "invalid payload length"), \ C(LOG, -ENXIO, "invalid or unsupported log page"), \ - C(INTERRUPTED, -ENXIO, "asynchronous event occured"), \ + C(INTERRUPTED, -ENXIO, "asynchronous event occurred"), \ C(FEATUREVERSION, -ENXIO, "unsupported feature version"), \ C(FEATURESELVALUE, -ENXIO, "unsupported feature selection value"), \ C(FEATURETRANSFERIP, -ENXIO, "feature transfer in progress"), \ @@ -431,7 +431,6 @@ static inline struct cxl_dev_state *mbox_to_cxlds(struct cxl_mailbox *cxl_mbox) * @poison: poison driver state info * @security: security driver state info * @fw: firmware upload / activation state - * @mce_notifier: MCE notifier * * See CXL 3.0 8.2.9.8.2 Capacity Configuration and Label Storage for * details on capacity parameters. @@ -451,7 +450,6 @@ struct cxl_memdev_state { struct cxl_poison_state poison; struct cxl_security_state security; struct cxl_fw_state fw; - struct notifier_block mce_notifier; }; static inline struct cxl_memdev_state * diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h index b826eb53cf7b..110ec9c44f09 100644 --- a/drivers/cxl/cxlpci.h +++ b/drivers/cxl/cxlpci.h @@ -13,16 +13,6 @@ */ #define CXL_PCI_DEFAULT_MAX_VECTORS 16 -/* Register Block Identifier (RBI) */ -enum cxl_regloc_type { - CXL_REGLOC_RBI_EMPTY = 0, - CXL_REGLOC_RBI_COMPONENT, - CXL_REGLOC_RBI_VIRT, - CXL_REGLOC_RBI_MEMDEV, - CXL_REGLOC_RBI_PMU, - CXL_REGLOC_RBI_TYPES -}; - /* * Table Access DOE, CDAT Read Entry Response * @@ -112,6 +102,4 @@ static inline void devm_cxl_port_ras_setup(struct cxl_port *port) } #endif -int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type, - struct cxl_register_map *map); #endif /* __CXL_PCI_H__ */ diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index 267c679b0b3c..90d2b7ee46b2 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -11,6 +11,7 @@ #include <linux/pci.h> #include <linux/aer.h> #include <linux/io.h> +#include <cxl/pci.h> #include <cxl/mailbox.h> #include "cxlmem.h" #include "cxlpci.h" @@ -691,12 +692,6 @@ static int cxl_pci_type3_init_mailbox(struct cxl_dev_state *cxlds) { int rc; - /* - * Fail the init if there's no mailbox. For a type3 this is out of spec. - */ - if (!cxlds->reg_map.device_map.mbox.valid) - return -ENODEV; - rc = cxl_mailbox_init(&cxlds->cxl_mbox, cxlds->dev); if (rc) return rc; diff --git a/include/cxl/cxl.h b/include/cxl/cxl.h index 016c74fb747c..802b143de83d 100644 --- a/include/cxl/cxl.h +++ b/include/cxl/cxl.h @@ -226,4 +226,6 @@ struct cxl_dev_state *_devm_cxl_dev_state_create(struct device *dev, struct cxl_memdev *devm_cxl_probe_mem(struct cxl_dev_state *cxlds, struct range *range); + +int cxl_set_capacity(struct cxl_dev_state *cxlds, u64 capacity); #endif /* __CXL_CXL_H__ */ diff --git a/include/cxl/pci.h b/include/cxl/pci.h new file mode 100644 index 000000000000..3e0000015871 --- /dev/null +++ b/include/cxl/pci.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2020 Intel Corporation. All rights reserved. */ + +#ifndef __CXL_CXL_PCI_H__ +#define __CXL_CXL_PCI_H__ + +/* Register Block Identifier (RBI) */ +enum cxl_regloc_type { + CXL_REGLOC_RBI_EMPTY = 0, + CXL_REGLOC_RBI_COMPONENT, + CXL_REGLOC_RBI_VIRT, + CXL_REGLOC_RBI_MEMDEV, + CXL_REGLOC_RBI_PMU, + CXL_REGLOC_RBI_TYPES +}; + +struct cxl_register_map; +struct pci_dev; + +int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type, + struct cxl_register_map *map); +#endif |