Messages in this thread | | | Date | Tue, 6 Dec 2022 18:41:34 -0800 | From | Dan Williams <> | Subject | RE: [PATCH v3 2/6] cxl/mbox: Add GET_POISON_LIST mailbox command |
| |
alison.schofield@ wrote: > From: Alison Schofield <alison.schofield@intel.com> > > CXL devices maintain a list of locations that are poisoned or result > in poison if the addresses are accessed by the host. > > Per the spec (CXL 3.0 8.2.9.8.4.1), the device returns this Poison > list as a set of Media Error Records that include the source of the > error, the starting device physical address and length. The length is > the number of adjacent DPAs in the record and is in units of 64 bytes. > > Retrieve the list and log each Media Error Record as a trace event of > type 'cxl_poison'. > > When the poison list is requested by region, include the region name > and uuid in the trace event. > > Signed-off-by: Alison Schofield <alison.schofield@intel.com> > --- > drivers/cxl/core/mbox.c | 81 +++++++++++++++++++++++++++++++++++++++++ > drivers/cxl/cxlmem.h | 37 +++++++++++++++++++ > 2 files changed, 118 insertions(+) > > diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c > index 0c90f13870a4..88f034e97812 100644 > --- a/drivers/cxl/core/mbox.c > +++ b/drivers/cxl/core/mbox.c > @@ -9,6 +9,9 @@ > > #include "core.h" > > +#define CREATE_TRACE_POINTS > +#include <trace/events/cxl.h> > + > static bool cxl_raw_allow_all; > > /** > @@ -752,6 +755,7 @@ int cxl_dev_state_identify(struct cxl_dev_state *cxlds) > { > /* See CXL 2.0 Table 175 Identify Memory Device Output Payload */ > struct cxl_mbox_identify id; > + __le32 val = 0; > int rc; > > rc = cxl_mbox_send_cmd(cxlds, CXL_MBOX_OP_IDENTIFY, NULL, 0, &id, > @@ -771,6 +775,9 @@ int cxl_dev_state_identify(struct cxl_dev_state *cxlds) > cxlds->lsa_size = le32_to_cpu(id.lsa_size); > memcpy(cxlds->firmware_version, id.fw_revision, sizeof(id.fw_revision)); > > + memcpy(&val, id.poison_list_max_mer, 3);
I see Jonathan already commented on the need for get_unaligned_le24() here, seconded.
> + cxlds->poison_max = min_t(u32, le32_to_cpu(val), CXL_POISON_LIST_MAX); > + > return 0; > } > EXPORT_SYMBOL_NS_GPL(cxl_dev_state_identify, CXL); > @@ -835,6 +842,79 @@ int cxl_mem_create_range_info(struct cxl_dev_state *cxlds) > } > EXPORT_SYMBOL_NS_GPL(cxl_mem_create_range_info, CXL); > > +static void cxl_trace_poison(struct cxl_mbox_poison_payload_out *po, > + struct cxl_region *cxlr, > + const char *memdev_name, > + const char *pcidev_name)
Type-safety please. Pass a 'struct cxl_memdev *' and 'struct pci_dev *'. Might need to be 'struct device *' instead of 'struct pci_dev *' depending on if this needs to be called from cxl_test, but anything is better than a non-specific string.
> +{ > + const char *region_name = cxlr ? dev_name(&cxlr->dev) : NULL;
...and push this conversion into the trace point.
> + struct cxl_region_params *p = cxlr ? &cxlr->params : NULL; > + uuid_t *uuid = p ? &p->uuid : NULL; > + u64 addr, dpa, overflow_t = 0; > + u8 source; > + u32 len; > + > + if (po->flags & CXL_POISON_FLAG_OVERFLOW) > + overflow_t = le64_to_cpu(po->overflow_timestamp); > + > + for (int i = 0; i < le16_to_cpu(po->count); i++) { > + len = le32_to_cpu(po->record[i].length) * CXL_POISON_LEN_MULT; > + addr = le64_to_cpu(po->record[i].address); > + source = addr & CXL_POISON_SOURCE_MASK; > + dpa = addr & CXL_POISON_START_MASK; > + > + trace_cxl_poison(memdev_name, pcidev_name, region_name, uuid, > + dpa, len, source, po->flags, overflow_t); > + } > +} > + > +int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len, > + struct cxl_region *cxlr) > +{ > + struct cxl_dev_state *cxlds = cxlmd->cxlds; > + const char *memdev_name = dev_name(&cxlmd->dev); > + const char *pcidev_name = dev_name(cxlds->dev); > + struct cxl_mbox_poison_payload_out *po; > + struct cxl_mbox_poison_payload_in pi; > + int nr_records = 0; > + int rc; > + > + po = kvmalloc(cxlds->payload_size, GFP_KERNEL); > + if (!po) > + return -ENOMEM; > + > + pi.offset = cpu_to_le64(offset); > + pi.length = cpu_to_le64(len); > + > + rc = mutex_lock_interruptible(&cxlds->poison_list_mutex);
So I do not know what this mutex is protecting if there is an allocation per cxl_mem_get_poison() invocation. Although I suspect that's somewhat wasteful. Just allocate one buffer at the beginning of time and then use the lock to protect that buffer.
Although, I wonder if this and Event handling should share locks and one preallocated buffer? Otherwise I do think it is important for Events and Poison handling to be able to make forward progress without needing to allocate up to a megabyte of memory at runtime. The other payload_size allocations are for one-off things that run at the beginning of time, but Poison and Events run repeatedly.
> + if (rc) > + goto out; > + > + do { > + rc = cxl_mbox_send_cmd(cxlds, CXL_MBOX_OP_GET_POISON, &pi, > + sizeof(pi), po, cxlds->payload_size); > + if (rc) > + break; > + > + if (trace_cxl_poison_enabled()) > + cxl_trace_poison(po, cxlr, memdev_name, pcidev_name); > + > + /* Protect against an uncleared _FLAG_MORE */ > + nr_records = nr_records + le16_to_cpu(po->count); > + if (nr_records >= cxlds->poison_max) { > + dev_dbg(&cxlmd->dev, "Max Error Records reached: %d\n", > + nr_records); > + break; > + } > + } while (po->flags & CXL_POISON_FLAG_MORE); > + > + mutex_unlock(&cxlds->poison_list_mutex); > +out: > + kvfree(po); > + return rc; > +} > +EXPORT_SYMBOL_NS_GPL(cxl_mem_get_poison, CXL); > + > struct cxl_dev_state *cxl_dev_state_create(struct device *dev) > { > struct cxl_dev_state *cxlds; > @@ -846,6 +926,7 @@ struct cxl_dev_state *cxl_dev_state_create(struct device *dev) > } > > mutex_init(&cxlds->mbox_mutex); > + mutex_init(&cxlds->poison_list_mutex); > cxlds->dev = dev; > > return cxlds; > diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h > index 669868cc1553..49d891347e39 100644 > --- a/drivers/cxl/cxlmem.h > +++ b/drivers/cxl/cxlmem.h > @@ -192,6 +192,8 @@ struct cxl_endpoint_dvsec_info { > * (CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register) > * @lsa_size: Size of Label Storage Area > * (CXL 2.0 8.2.9.5.1.1 Identify Memory Device) > + * @poison_max: maximum media error records held in device cache > + * @poison_list_mutex: Mutex to synchronize poison list retrieval > * @mbox_mutex: Mutex to synchronize mailbox access. > * @firmware_version: Firmware version for the memory device. > * @enabled_cmds: Hardware commands found enabled in CEL. > @@ -224,6 +226,8 @@ struct cxl_dev_state { > > size_t payload_size; > size_t lsa_size; > + u32 poison_max; > + struct mutex poison_list_mutex; /* Protect reads of poison list */ > struct mutex mbox_mutex; /* Protects device mailbox and firmware */ > char firmware_version[0x10]; > DECLARE_BITMAP(enabled_cmds, CXL_MEM_COMMAND_ID_MAX); > @@ -349,6 +353,37 @@ struct cxl_mbox_set_partition_info { > > /* Get Poison List CXL 3.0 Spec 8.2.9.8.4.1 */ > > +struct cxl_mbox_poison_payload_in { > + __le64 offset; > + __le64 length; > +} __packed; > + > +struct cxl_mbox_poison_payload_out { > + u8 flags; > + u8 rsvd1; > + __le64 overflow_timestamp; > + __le16 count; > + u8 rsvd2[0x14];
Let's use decimal values for size.
> + struct cxl_poison_record { > + __le64 address; > + __le32 length; > + __le32 rsvd; > + } __packed record[]; > +} __packed; > + > +/* > + * Get Poison List address field encodes the starting > + * address of poison, and the source of the poison. > + */ > +#define CXL_POISON_START_MASK GENMASK_ULL(63, 6) > +#define CXL_POISON_SOURCE_MASK GENMASK(2, 0) > + > +/* Get Poison List record length is in units of 64 bytes */ > +#define CXL_POISON_LEN_MULT 64 > + > +/* Kernel defined maximum for a list of poison errors */ > +#define CXL_POISON_LIST_MAX 1024 > + > /* Get Poison List: Payload out flags */ > #define CXL_POISON_FLAG_MORE BIT(0) > #define CXL_POISON_FLAG_OVERFLOW BIT(1) > @@ -395,6 +430,8 @@ int cxl_mem_create_range_info(struct cxl_dev_state *cxlds); > struct cxl_dev_state *cxl_dev_state_create(struct device *dev); > void set_exclusive_cxl_commands(struct cxl_dev_state *cxlds, unsigned long *cmds); > void clear_exclusive_cxl_commands(struct cxl_dev_state *cxlds, unsigned long *cmds); > +int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len, > + struct cxl_region *cxlr); > #ifdef CONFIG_CXL_SUSPEND > void cxl_mem_active_inc(void); > void cxl_mem_active_dec(void); > -- > 2.37.3 >
| |