Messages in this thread Patch in this message | | | From | Abhijit Gangurde <> | Subject | [PATCH v5 3/5] cdx: add support for bus enable and disable | Date | Fri, 22 Sep 2023 15:48:26 +0530 |
| |
CDX bus needs to be disabled before updating/writing devices in the FPGA. Once the devices are written, the bus shall be rescanned. This change provides sysfs entry to enable/disable the CDX bus.
Co-developed-by: Nipun Gupta <nipun.gupta@amd.com> Signed-off-by: Nipun Gupta <nipun.gupta@amd.com> Signed-off-by: Abhijit Gangurde <abhijit.gangurde@amd.com> Reviewed-by: Pieter Jansen van Vuuren <pieter.jansen-van-vuuren@amd.com> Tested-by: Nikhil Agarwal <nikhil.agarwal@amd.com> --- Changes in v5: - Rebased on Linux 6.6
Changes in v4: - Add sparse marking for lock around cdx_find_controller() function
Changes in v3: - Use platform bus to locate controller
Changes in v2: - Improve documentation - Split sysfs entry enable to enable and disable - sysfs entry enable and disable take bus number as an argument
Documentation/ABI/testing/sysfs-bus-cdx | 27 +++++++ drivers/cdx/cdx.c | 97 +++++++++++++++++++++++++ drivers/cdx/controller/cdx_controller.c | 50 +++++++++++++ drivers/cdx/controller/mc_cdx_pcol.h | 54 ++++++++++++++ drivers/cdx/controller/mcdi_functions.c | 24 ++++++ drivers/cdx/controller/mcdi_functions.h | 18 +++++ include/linux/cdx/cdx_bus.h | 12 +++ 7 files changed, 282 insertions(+)
diff --git a/Documentation/ABI/testing/sysfs-bus-cdx b/Documentation/ABI/testing/sysfs-bus-cdx index 7af477f49998..04c8dfe7e201 100644 --- a/Documentation/ABI/testing/sysfs-bus-cdx +++ b/Documentation/ABI/testing/sysfs-bus-cdx @@ -11,6 +11,33 @@ Description: # echo 1 > /sys/bus/cdx/rescan +What: /sys/bus/cdx/enable +Date: July 2023 +Contact: nipun.gupta@amd.com +Description: + Writing bus number in hex to this file will attempt to enable + the CDX bus. The bus number for the cdx devices can be found + at /sys/bus/cdx/devices/cdx-BB:DD, where BB denotes the bus + number for the respective device. + + For example :: + + # echo 00 > /sys/bus/cdx/enable + +What: /sys/bus/cdx/disable +Date: July 2023 +Contact: nipun.gupta@amd.com +Description: + Writing bus number in hex to this file will attempt to disable + the CDX bus. CDX bus should be disabled before updating the + devices in FPGA. The bus number for the cdx devices can be + found at /sys/bus/cdx/devices/cdx-BB:DD, where BB denotes the + bus number for the respective device. + + For example :: + + # echo 00 > /sys/bus/cdx/disable + What: /sys/bus/cdx/devices/.../vendor Date: March 2023 Contact: nipun.gupta@amd.com diff --git a/drivers/cdx/cdx.c b/drivers/cdx/cdx.c index cbfe9a2f3646..8da043756fa4 100644 --- a/drivers/cdx/cdx.c +++ b/drivers/cdx/cdx.c @@ -70,6 +70,10 @@ #define CDX_DEFAULT_DMA_MASK (~0ULL) #define MAX_CDX_CONTROLLERS 16 +#define CONTROLLER_ID(X) \ + (((X) & CDX_CONTROLLER_ID_MASK) >> CDX_CONTROLLER_ID_SHIFT) +#define BUS_ID(X) ((X) & CDX_BUS_NUM_MASK) + /* IDA for CDX controllers registered with the CDX bus */ static DEFINE_IDA(cdx_controller_ida); /* Lock to protect controller ops */ @@ -384,6 +388,97 @@ static struct attribute *cdx_dev_attrs[] = { }; ATTRIBUTE_GROUPS(cdx_dev); +/* Must be called with cdx_controller_lock acquired. + */ +static struct cdx_controller *cdx_find_controller(u8 controller_id) + __must_hold(&cdx_controller_lock) +{ + struct cdx_controller *cdx; + struct platform_device *pd; + struct device_node *np; + + for_each_compatible_node(np, NULL, compat_node_name) { + if (!np) + return NULL; + + pd = of_find_device_by_node(np); + if (!pd) + return NULL; + + cdx = platform_get_drvdata(pd); + if (cdx && cdx->controller_registered && cdx->id == controller_id) { + put_device(&pd->dev); + return cdx; + } + + put_device(&pd->dev); + } + + return NULL; +} + +static ssize_t enable_store(const struct bus_type *bus, + const char *buf, size_t count) +{ + unsigned long controller_id; + struct cdx_controller *cdx; + u8 bus_id; + int ret; + + if (kstrtou8(buf, 16, &bus_id)) + return -EINVAL; + + controller_id = CONTROLLER_ID(bus_id); + bus_id = BUS_ID(bus_id); + + mutex_lock(&cdx_controller_lock); + + cdx = cdx_find_controller(controller_id); + if (cdx) + if (cdx->ops->bus_enable) + ret = cdx->ops->bus_enable(cdx, bus_id); + else + ret = -EOPNOTSUPP; + else + ret = -EINVAL; + + mutex_unlock(&cdx_controller_lock); + + return ret < 0 ? ret : count; +} +static BUS_ATTR_WO(enable); + +static ssize_t disable_store(const struct bus_type *bus, + const char *buf, size_t count) +{ + unsigned long controller_id; + struct cdx_controller *cdx; + u8 bus_id; + int ret; + + if (kstrtou8(buf, 16, &bus_id)) + return -EINVAL; + + controller_id = CONTROLLER_ID(bus_id); + bus_id = BUS_ID(bus_id); + + mutex_lock(&cdx_controller_lock); + + cdx = cdx_find_controller(controller_id); + if (cdx) + if (cdx->ops->bus_disable) + ret = cdx->ops->bus_disable(cdx, bus_id); + else + ret = -EOPNOTSUPP; + else + ret = -EINVAL; + + mutex_unlock(&cdx_controller_lock); + + return ret < 0 ? ret : count; +} +static BUS_ATTR_WO(disable); + static ssize_t rescan_store(const struct bus_type *bus, const char *buf, size_t count) { @@ -426,6 +521,8 @@ static ssize_t rescan_store(const struct bus_type *bus, static BUS_ATTR_WO(rescan); static struct attribute *cdx_bus_attrs[] = { + &bus_attr_enable.attr, + &bus_attr_disable.attr, &bus_attr_rescan.attr, NULL, }; diff --git a/drivers/cdx/controller/cdx_controller.c b/drivers/cdx/controller/cdx_controller.c index bb4ae7970e21..78bc73e787b6 100644 --- a/drivers/cdx/controller/cdx_controller.c +++ b/drivers/cdx/controller/cdx_controller.c @@ -33,6 +33,42 @@ static const struct cdx_mcdi_ops mcdi_ops = { .mcdi_request = cdx_mcdi_request, }; +static int cdx_bus_enable(struct cdx_controller *cdx, u8 bus_num) +{ + int ret; + + if (test_bit(bus_num, cdx->bus_state)) + return 0; + + ret = cdx_mcdi_bus_enable(cdx->priv, bus_num); + if (!ret) + set_bit(bus_num, cdx->bus_state); + + return ret; +} + +static int cdx_bus_disable(struct cdx_controller *cdx, u8 bus_num) +{ + int ret; + + if (!test_bit(bus_num, cdx->bus_state)) + return 0; + + ret = cdx_mcdi_bus_disable(cdx->priv, bus_num); + if (!ret) + clear_bit(bus_num, cdx->bus_state); + + return ret; +} + +static void cdx_bus_disable_all(struct cdx_controller *cdx) +{ + u8 bus_num; + + for_each_set_bit(bus_num, cdx->bus_state, MAX_CDX_BUSES) + cdx_bus_disable(cdx, bus_num); +} + void cdx_rpmsg_post_probe(struct cdx_controller *cdx) { /* Register CDX controller with CDX bus driver */ @@ -43,6 +79,7 @@ void cdx_rpmsg_post_probe(struct cdx_controller *cdx) void cdx_rpmsg_pre_remove(struct cdx_controller *cdx) { cdx_unregister_controller(cdx); + cdx_bus_disable_all(cdx); cdx_mcdi_wait_for_quiescence(cdx->priv, MCDI_RPC_TIMEOUT); } @@ -81,11 +118,22 @@ static int cdx_scan_devices(struct cdx_controller *cdx) for (bus_num = 0; bus_num < num_cdx_bus; bus_num++) { u8 num_cdx_dev; + ret = cdx_bus_enable(cdx, bus_num); + if (ret && ret != -EALREADY) { + dev_err(cdx->dev, + "CDX bus %d enable failed: %d\n", bus_num, ret); + continue; + } + /* MCDI FW Read: Fetch the number of devices present */ ret = cdx_mcdi_get_num_devs(cdx_mcdi, bus_num); if (ret < 0) { dev_err(cdx->dev, "Get devices on CDX bus %d failed: %d\n", bus_num, ret); + ret = cdx_bus_disable(cdx, bus_num); + if (ret) + dev_err(cdx->dev, + "CDX bus %d disable failed: %d\n", bus_num, ret); continue; } num_cdx_dev = (u8)ret; @@ -121,6 +169,8 @@ static int cdx_scan_devices(struct cdx_controller *cdx) } static struct cdx_ops cdx_ops = { + .bus_enable = cdx_bus_enable, + .bus_disable = cdx_bus_disable, .scan = cdx_scan_devices, .dev_configure = cdx_configure_device, }; diff --git a/drivers/cdx/controller/mc_cdx_pcol.h b/drivers/cdx/controller/mc_cdx_pcol.h index 4ccb7b52951b..2de019406b57 100644 --- a/drivers/cdx/controller/mc_cdx_pcol.h +++ b/drivers/cdx/controller/mc_cdx_pcol.h @@ -455,6 +455,60 @@ #define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_REQUESTER_ID_OFST 84 #define MC_CMD_CDX_BUS_GET_DEVICE_CONFIG_OUT_REQUESTER_ID_LEN 4 +/***********************************/ +/* + * MC_CMD_CDX_BUS_DOWN + * Asserting reset on the CDX bus causes all devices on the bus to be quiesced. + * DMA bus mastering is disabled and any pending DMA request are flushed. Once + * the response is returned, the devices are guaranteed to no longer issue DMA + * requests or raise MSI interrupts. Further device MMIO accesses may have + * undefined results. While the bus reset is asserted, any of the enumeration + * or device configuration MCDIs will fail with EAGAIN. It is only legal to + * reload the relevant PL region containing CDX devices if the corresponding CDX + * bus is in reset. Depending on the implementation, the firmware may or may + * not enforce this restriction and it is up to the caller to make sure this + * requirement is satisfied. + */ +#define MC_CMD_CDX_BUS_DOWN 0x4 +#define MC_CMD_CDX_BUS_DOWN_MSGSET 0x4 + +/* MC_CMD_CDX_BUS_DOWN_IN msgrequest */ +#define MC_CMD_CDX_BUS_DOWN_IN_LEN 4 +/* Bus number to put in reset, in range 0 to BUS_COUNT-1 */ +#define MC_CMD_CDX_BUS_DOWN_IN_BUS_OFST 0 +#define MC_CMD_CDX_BUS_DOWN_IN_BUS_LEN 4 + +/* + * MC_CMD_CDX_BUS_DOWN_OUT msgresponse: The bus is quiesced, no further + * upstream traffic for devices on this bus. + */ +#define MC_CMD_CDX_BUS_DOWN_OUT_LEN 0 + +/***********************************/ +/* + * MC_CMD_CDX_BUS_UP + * After bus reset is de-asserted, devices are in a state which is functionally + * equivalent to each device having been reset with MC_CMD_CDX_DEVICE_RESET. In + * other words, device logic is reset in a hardware-specific way, MMIO accesses + * are forwarded to the device, DMA bus mastering is disabled and needs to be + * re-enabled with MC_CMD_CDX_DEVICE_DMA_ENABLE once the driver is ready to + * start servicing DMA. If the underlying number of devices or device resources + * changed (e.g. if PL was reloaded) while the bus was in reset, the bus driver + * is expected to re-enumerate the bus. Returns EALREADY if the bus was already + * up before the call. + */ +#define MC_CMD_CDX_BUS_UP 0x5 +#define MC_CMD_CDX_BUS_UP_MSGSET 0x5 + +/* MC_CMD_CDX_BUS_UP_IN msgrequest */ +#define MC_CMD_CDX_BUS_UP_IN_LEN 4 +/* Bus number to take out of reset, in range 0 to BUS_COUNT-1 */ +#define MC_CMD_CDX_BUS_UP_IN_BUS_OFST 0 +#define MC_CMD_CDX_BUS_UP_IN_BUS_LEN 4 + +/* MC_CMD_CDX_BUS_UP_OUT msgresponse: The bus can now be enumerated. */ +#define MC_CMD_CDX_BUS_UP_OUT_LEN 0 + /***********************************/ /* * MC_CMD_CDX_DEVICE_RESET diff --git a/drivers/cdx/controller/mcdi_functions.c b/drivers/cdx/controller/mcdi_functions.c index 0158f26533dd..0e1e35d91242 100644 --- a/drivers/cdx/controller/mcdi_functions.c +++ b/drivers/cdx/controller/mcdi_functions.c @@ -124,6 +124,30 @@ int cdx_mcdi_get_dev_config(struct cdx_mcdi *cdx, return 0; } +int cdx_mcdi_bus_enable(struct cdx_mcdi *cdx, u8 bus_num) +{ + MCDI_DECLARE_BUF(inbuf, MC_CMD_CDX_BUS_UP_IN_LEN); + int ret; + + MCDI_SET_DWORD(inbuf, CDX_BUS_UP_IN_BUS, bus_num); + ret = cdx_mcdi_rpc(cdx, MC_CMD_CDX_BUS_UP, inbuf, sizeof(inbuf), + NULL, 0, NULL); + + return ret; +} + +int cdx_mcdi_bus_disable(struct cdx_mcdi *cdx, u8 bus_num) +{ + MCDI_DECLARE_BUF(inbuf, MC_CMD_CDX_BUS_DOWN_IN_LEN); + int ret; + + MCDI_SET_DWORD(inbuf, CDX_BUS_DOWN_IN_BUS, bus_num); + ret = cdx_mcdi_rpc(cdx, MC_CMD_CDX_BUS_DOWN, inbuf, sizeof(inbuf), + NULL, 0, NULL); + + return ret; +} + int cdx_mcdi_reset_device(struct cdx_mcdi *cdx, u8 bus_num, u8 dev_num) { MCDI_DECLARE_BUF(inbuf, MC_CMD_CDX_DEVICE_RESET_IN_LEN); diff --git a/drivers/cdx/controller/mcdi_functions.h b/drivers/cdx/controller/mcdi_functions.h index 7440ace5539a..28973d5ec3ab 100644 --- a/drivers/cdx/controller/mcdi_functions.h +++ b/drivers/cdx/controller/mcdi_functions.h @@ -47,6 +47,24 @@ int cdx_mcdi_get_dev_config(struct cdx_mcdi *cdx, u8 bus_num, u8 dev_num, struct cdx_dev_params *dev_params); +/** + * cdx_mcdi_bus_enable - Enable CDX bus represented by bus_num + * @cdx: pointer to MCDI interface. + * @bus_num: Bus number. + * + * Return: 0 on success, <0 on failure + */ +int cdx_mcdi_bus_enable(struct cdx_mcdi *cdx, u8 bus_num); + +/** + * cdx_mcdi_bus_disable - Disable CDX bus represented by bus_num + * @cdx: pointer to MCDI interface. + * @bus_num: Bus number. + * + * Return: 0 on success, <0 on failure + */ +int cdx_mcdi_bus_disable(struct cdx_mcdi *cdx, u8 bus_num); + /** * cdx_mcdi_reset_device - Reset cdx device represented by bus_num:dev_num * @cdx: pointer to MCDI interface. diff --git a/include/linux/cdx/cdx_bus.h b/include/linux/cdx/cdx_bus.h index 82c27b8c94e1..ac715bc84315 100644 --- a/include/linux/cdx/cdx_bus.h +++ b/include/linux/cdx/cdx_bus.h @@ -16,6 +16,8 @@ #define MAX_CDX_DEV_RESOURCES 4 #define CDX_CONTROLLER_ID_SHIFT 4 #define CDX_BUS_NUM_MASK 0xF +#define CDX_CONTROLLER_ID_MASK 0xF0 +#define MAX_CDX_BUSES (CDX_BUS_NUM_MASK + 1) /* Forward declaration for CDX controller */ struct cdx_controller; @@ -28,6 +30,10 @@ struct cdx_device_config { u8 type; }; +typedef int (*cdx_bus_enable_cb)(struct cdx_controller *cdx, u8 bus_num); + +typedef int (*cdx_bus_disable_cb)(struct cdx_controller *cdx, u8 bus_num); + typedef int (*cdx_scan_cb)(struct cdx_controller *cdx); typedef int (*cdx_dev_configure_cb)(struct cdx_controller *cdx, @@ -49,11 +55,15 @@ typedef int (*cdx_dev_configure_cb)(struct cdx_controller *cdx, /** * struct cdx_ops - Callbacks supported by CDX controller. + * @bus_enable: enable bus on the controller + * @bus_disable: disable bus on the controller * @scan: scan the devices on the controller * @dev_configure: configuration like reset, master_enable, * msi_config etc for a CDX device */ struct cdx_ops { + cdx_bus_enable_cb bus_enable; + cdx_bus_disable_cb bus_disable; cdx_scan_cb scan; cdx_dev_configure_cb dev_configure; }; @@ -64,6 +74,7 @@ struct cdx_ops { * @priv: private data * @id: Controller ID * @controller_registered: controller registered with bus + * @bus_state: state of the buses(enabled/disabled) * @ops: CDX controller ops */ struct cdx_controller { @@ -71,6 +82,7 @@ struct cdx_controller { void *priv; u32 id; bool controller_registered; + DECLARE_BITMAP(bus_state, MAX_CDX_BUSES); struct cdx_ops *ops; }; -- 2.25.1
| |