Skip to content

L
linux
Commit 9ac543c0 authored by Jakub Kicinski's avatar Jakub Kicinski
Browse files
Options

Merge branch 'aux-bus-v11' of https://github.com/ajitkhaparde1/linux 

Ajit Khaparde says:

====================
bnxt: Add Auxiliary driver support

Add auxiliary device driver for Broadcom devices.
The bnxt_en driver will register and initialize an aux device
if RDMA is enabled in the underlying device.
The bnxt_re driver will then probe and initialize the
RoCE interfaces with the infiniband stack.

We got rid of the bnxt_en_ops which the bnxt_re driver used to
communicate with bnxt_en.
Similarly  We have tried to clean up most of the bnxt_ulp_ops.
In most of the cases we used the functions and entry points provided
by the auxiliary bus driver framework.
And now these are the minimal functions needed to support the functionality.

We will try to work on getting rid of the remaining if we find any
other viable option in future.

* 'aux-bus-v11' of https://github.com/ajitkhaparde1/linux:
  bnxt_en: Remove runtime interrupt vector allocation
  RDMA/bnxt_re: Remove the sriov config callback
  bnxt_en: Remove struct bnxt access from RoCE driver
  bnxt_en: Use auxiliary bus calls over proprietary calls
  bnxt_en: Use direct API instead of indirection
  bnxt_en: Remove usage of ulp_id
  RDMA/bnxt_re: Use auxiliary driver interface
  bnxt_en: Add auxiliary driver support
====================

Link: https://lore.kernel.org/r/20230202033809.3989-1-ajit.khaparde@broadcom.comSigned-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents c21adf25 30343221
Hide whitespace changes
Inline Side-by-side
Showing with 461 additions and 735 deletions
drivers/infiniband/hw/bnxt_re/bnxt_re.h
View file @ 9ac543c0
......@@ -89,13 +89,6 @@ struct bnxt_re_ring_attr {
u8 mode;
};
struct bnxt_re_work {
struct work_struct work;
unsigned long event;
struct bnxt_re_dev *rdev;
struct net_device *vlan_dev;
};
struct bnxt_re_sqp_entries {
struct bnxt_qplib_sge sge;
u64 wrid;
......@@ -132,10 +125,10 @@ struct bnxt_re_dev {
#define BNXT_RE_FLAG_ERR_DEVICE_DETACHED 17
#define BNXT_RE_FLAG_ISSUE_ROCE_STATS 29
struct net_device *netdev;
struct notifier_block nb;
unsigned int version, major, minor;
struct bnxt_qplib_chip_ctx *chip_ctx;
struct bnxt_en_dev *en_dev;
struct bnxt_msix_entry msix_entries[BNXT_RE_MAX_MSIX];
int num_msix;
int id;
......@@ -194,5 +187,4 @@ static inline struct device *rdev_to_dev(struct bnxt_re_dev *rdev)
return &rdev->ibdev.dev;
return NULL;
}
#endif
drivers/infiniband/hw/bnxt_re/main.c
View file @ 9ac543c0
......@@ -48,6 +48,7 @@
#include <net/ipv6.h>
#include <net/addrconf.h>
#include <linux/if_ether.h>
#include <linux/auxiliary_bus.h>
#include <rdma/ib_verbs.h>
#include <rdma/ib_user_verbs.h>
......@@ -74,14 +75,14 @@ MODULE_DESCRIPTION(BNXT_RE_DESC " Driver");
MODULE_LICENSE("Dual BSD/GPL");
/* globals */
static struct list_head bnxt_re_dev_list = LIST_HEAD_INIT(bnxt_re_dev_list);
/* Mutex to protect the list of bnxt_re devices added */
static DEFINE_MUTEX(bnxt_re_dev_lock);
static struct workqueue_struct *bnxt_re_wq;
static void bnxt_re_remove_device(struct bnxt_re_dev *rdev);
static void bnxt_re_dealloc_driver(struct ib_device *ib_dev);
static DEFINE_MUTEX(bnxt_re_mutex);
static void bnxt_re_stop_irq(void *handle);
static void bnxt_re_dev_stop(struct bnxt_re_dev *rdev);
static int bnxt_re_netdev_event(struct notifier_block *notifier,
unsigned long event, void *ptr);
static struct bnxt_re_dev *bnxt_re_from_netdev(struct net_device *netdev);
static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev);
static void bnxt_re_set_drv_mode(struct bnxt_re_dev *rdev, u8 mode)
{
......@@ -111,16 +112,14 @@ static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev, u8 wqe_mode)
{
struct bnxt_qplib_chip_ctx *chip_ctx;
struct bnxt_en_dev *en_dev;
struct bnxt *bp;
en_dev = rdev->en_dev;
bp = netdev_priv(en_dev->net);
chip_ctx = kzalloc(sizeof(*chip_ctx), GFP_KERNEL);
if (!chip_ctx)
return -ENOMEM;
chip_ctx->chip_num = bp->chip_num;
chip_ctx->hw_stats_size = bp->hw_ring_stats_size;
chip_ctx->chip_num = en_dev->chip_num;
chip_ctx->hw_stats_size = en_dev->hw_ring_stats_size;
rdev->chip_ctx = chip_ctx;
/* rest members to follow eventually */
......@@ -128,7 +127,7 @@ static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev, u8 wqe_mode)
rdev->qplib_res.cctx = rdev->chip_ctx;
rdev->rcfw.res = &rdev->qplib_res;
rdev->qplib_res.dattr = &rdev->dev_attr;
rdev->qplib_res.is_vf = BNXT_VF(bp);
rdev->qplib_res.is_vf = BNXT_EN_VF(en_dev);
bnxt_re_set_drv_mode(rdev, wqe_mode);
if (bnxt_qplib_determine_atomics(en_dev->pdev))
......@@ -141,10 +140,7 @@ static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev, u8 wqe_mode)
static void bnxt_re_get_sriov_func_type(struct bnxt_re_dev *rdev)
{
struct bnxt *bp;
bp = netdev_priv(rdev->en_dev->net);
if (BNXT_VF(bp))
if (BNXT_EN_VF(rdev->en_dev))
rdev->is_virtfn = 1;
}
......@@ -225,56 +221,12 @@ static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev)
bnxt_re_limit_vf_res(&rdev->qplib_ctx, num_vfs);
}
/* for handling bnxt_en callbacks later */
static void bnxt_re_stop(void *p)
{
struct bnxt_re_dev *rdev = p;
struct bnxt *bp;
if (!rdev)
return;
ASSERT_RTNL();
/* L2 driver invokes this callback during device error/crash or device
* reset. Current RoCE driver doesn't recover the device in case of
* error. Handle the error by dispatching fatal events to all qps
* ie. by calling bnxt_re_dev_stop and release the MSIx vectors as
* L2 driver want to modify the MSIx table.
*/
bp = netdev_priv(rdev->netdev);
ibdev_info(&rdev->ibdev, "Handle device stop call from L2 driver");
/* Check the current device state from L2 structure and move the
* device to detached state if FW_FATAL_COND is set.
* This prevents more commands to HW during clean-up,
* in case the device is already in error.
*/
if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags);
bnxt_re_dev_stop(rdev);
bnxt_re_stop_irq(rdev);
/* Move the device states to detached and avoid sending any more
* commands to HW
*/
set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags);
set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags);
}
static void bnxt_re_start(void *p)
{
}
static void bnxt_re_sriov_config(void *p, int num_vfs)
static void bnxt_re_vf_res_config(struct bnxt_re_dev *rdev)
{
struct bnxt_re_dev *rdev = p;
if (!rdev)
return;
if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
return;
rdev->num_vfs = num_vfs;
rdev->num_vfs = pci_sriov_get_totalvfs(rdev->en_dev->pdev);
if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx)) {
bnxt_re_set_resource_limits(rdev);
bnxt_qplib_set_func_resources(&rdev->qplib_res, &rdev->rcfw,
......@@ -282,16 +234,14 @@ static void bnxt_re_sriov_config(void *p, int num_vfs)
}
}
static void bnxt_re_shutdown(void *p)
static void bnxt_re_shutdown(struct auxiliary_device *adev)
{
struct bnxt_re_dev *rdev = p;
struct bnxt_re_dev *rdev = auxiliary_get_drvdata(adev);
if (!rdev)
return;
ASSERT_RTNL();
/* Release the MSIx vectors before queuing unregister */
bnxt_re_stop_irq(rdev);
ib_unregister_device_queued(&rdev->ibdev);
ib_unregister_device(&rdev->ibdev);
bnxt_re_dev_uninit(rdev);
}
static void bnxt_re_stop_irq(void *handle)
......@@ -312,7 +262,7 @@ static void bnxt_re_stop_irq(void *handle)
static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent)
{
struct bnxt_re_dev *rdev = (struct bnxt_re_dev *)handle;
struct bnxt_msix_entry *msix_ent = rdev->msix_entries;
struct bnxt_msix_entry *msix_ent = rdev->en_dev->msix_entries;
struct bnxt_qplib_rcfw *rcfw = &rdev->rcfw;
struct bnxt_qplib_nq *nq;
int indx, rc;
......@@ -331,7 +281,7 @@ static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent)
* in device sctructure.
*/
for (indx = 0; indx < rdev->num_msix; indx++)
rdev->msix_entries[indx].vector = ent[indx].vector;
rdev->en_dev->msix_entries[indx].vector = ent[indx].vector;
bnxt_qplib_rcfw_start_irq(rcfw, msix_ent[BNXT_RE_AEQ_IDX].vector,
false);
......@@ -346,93 +296,22 @@ static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent)
}
static struct bnxt_ulp_ops bnxt_re_ulp_ops = {
.ulp_async_notifier = NULL,
.ulp_stop = bnxt_re_stop,
.ulp_start = bnxt_re_start,
.ulp_sriov_config = bnxt_re_sriov_config,
.ulp_shutdown = bnxt_re_shutdown,
.ulp_irq_stop = bnxt_re_stop_irq,
.ulp_irq_restart = bnxt_re_start_irq
};
/* RoCE -> Net driver */
/* Driver registration routines used to let the networking driver (bnxt_en)
* to know that the RoCE driver is now installed
*/
static int bnxt_re_unregister_netdev(struct bnxt_re_dev *rdev)
{
struct bnxt_en_dev *en_dev;
int rc;
if (!rdev)
return -EINVAL;
en_dev = rdev->en_dev;
rc = en_dev->en_ops->bnxt_unregister_device(rdev->en_dev,
BNXT_ROCE_ULP);
return rc;
}
static int bnxt_re_register_netdev(struct bnxt_re_dev *rdev)
{
struct bnxt_en_dev *en_dev;
int rc = 0;
if (!rdev)
return -EINVAL;
en_dev = rdev->en_dev;
rc = en_dev->en_ops->bnxt_register_device(en_dev, BNXT_ROCE_ULP,
&bnxt_re_ulp_ops, rdev);
rdev->qplib_res.pdev = rdev->en_dev->pdev;
return rc;
}
static int bnxt_re_free_msix(struct bnxt_re_dev *rdev)
{
struct bnxt_en_dev *en_dev;
int rc;
if (!rdev)
return -EINVAL;
en_dev = rdev->en_dev;
rc = en_dev->en_ops->bnxt_free_msix(rdev->en_dev, BNXT_ROCE_ULP);
return rc;
}
static int bnxt_re_request_msix(struct bnxt_re_dev *rdev)
{
int rc = 0, num_msix_want = BNXT_RE_MAX_MSIX, num_msix_got;
struct bnxt_en_dev *en_dev;
if (!rdev)
return -EINVAL;
en_dev = rdev->en_dev;
num_msix_want = min_t(u32, BNXT_RE_MAX_MSIX, num_online_cpus());
num_msix_got = en_dev->en_ops->bnxt_request_msix(en_dev, BNXT_ROCE_ULP,
rdev->msix_entries,
num_msix_want);
if (num_msix_got < BNXT_RE_MIN_MSIX) {
rc = -EINVAL;
goto done;
}
if (num_msix_got != num_msix_want) {
ibdev_warn(&rdev->ibdev,
"Requested %d MSI-X vectors, got %d\n",
num_msix_want, num_msix_got);
}
rdev->num_msix = num_msix_got;
done:
rc = bnxt_register_dev(en_dev, &bnxt_re_ulp_ops, rdev);
if (!rc)
rdev->qplib_res.pdev = rdev->en_dev->pdev;
return rc;
}
......@@ -458,12 +337,17 @@ static void bnxt_re_fill_fw_msg(struct bnxt_fw_msg *fw_msg, void *msg,
static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev,
u16 fw_ring_id, int type)
{
struct bnxt_en_dev *en_dev = rdev->en_dev;
struct bnxt_en_dev *en_dev;
struct hwrm_ring_free_input req = {0};
struct hwrm_ring_free_output resp;
struct bnxt_fw_msg fw_msg;
int rc = -EINVAL;
if (!rdev)
return rc;
en_dev = rdev->en_dev;
if (!en_dev)
return rc;
......@@ -477,7 +361,7 @@ static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev,
req.ring_id = cpu_to_le16(fw_ring_id);
bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
rc = bnxt_send_msg(en_dev, &fw_msg);
if (rc)
ibdev_err(&rdev->ibdev, "Failed to free HW ring:%d :%#x",
req.ring_id, rc);
......@@ -514,7 +398,7 @@ static int bnxt_re_net_ring_alloc(struct bnxt_re_dev *rdev,
req.int_mode = ring_attr->mode;
bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
rc = bnxt_send_msg(en_dev, &fw_msg);
if (!rc)
*fw_ring_id = le16_to_cpu(resp.ring_id);
......@@ -542,7 +426,7 @@ static int bnxt_re_net_stats_ctx_free(struct bnxt_re_dev *rdev,
req.stat_ctx_id = cpu_to_le32(fw_stats_ctx_id);
bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
rc = bnxt_send_msg(en_dev, &fw_msg);
if (rc)
ibdev_err(&rdev->ibdev, "Failed to free HW stats context %#x",
rc);
......@@ -575,7 +459,7 @@ static int bnxt_re_net_stats_ctx_alloc(struct bnxt_re_dev *rdev,
req.stat_ctx_flags = STAT_CTX_ALLOC_REQ_STAT_CTX_FLAGS_ROCE;
bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
rc = bnxt_send_msg(en_dev, &fw_msg);
if (!rc)
*fw_stats_ctx_id = le32_to_cpu(resp.stat_ctx_id);
......@@ -584,21 +468,6 @@ static int bnxt_re_net_stats_ctx_alloc(struct bnxt_re_dev *rdev,
/* Device */
static bool is_bnxt_re_dev(struct net_device *netdev)
{
struct ethtool_drvinfo drvinfo;
if (netdev->ethtool_ops && netdev->ethtool_ops->get_drvinfo) {
memset(&drvinfo, 0, sizeof(drvinfo));
netdev->ethtool_ops->get_drvinfo(netdev, &drvinfo);
if (strcmp(drvinfo.driver, "bnxt_en"))
return false;
return true;
}
return false;
}
static struct bnxt_re_dev *bnxt_re_from_netdev(struct net_device *netdev)
{
struct ib_device *ibdev =
......@@ -609,31 +478,6 @@ static struct bnxt_re_dev *bnxt_re_from_netdev(struct net_device *netdev)
return container_of(ibdev, struct bnxt_re_dev, ibdev);
}
static struct bnxt_en_dev *bnxt_re_dev_probe(struct net_device *netdev)
{
struct bnxt_en_dev *en_dev;
struct pci_dev *pdev;
en_dev = bnxt_ulp_probe(netdev);
if (IS_ERR(en_dev))
return en_dev;
pdev = en_dev->pdev;
if (!pdev)
return ERR_PTR(-EINVAL);
if (!(en_dev->flags & BNXT_EN_FLAG_ROCE_CAP)) {
dev_info(&pdev->dev,
"%s: probe error: RoCE is not supported on this device",
ROCE_DRV_MODULE_NAME);
return ERR_PTR(-ENODEV);
}
dev_hold(netdev);
return en_dev;
}
static ssize_t hw_rev_show(struct device *device, struct device_attribute *attr,
char *buf)
{
......@@ -679,7 +523,6 @@ static const struct ib_device_ops bnxt_re_dev_ops = {
.create_qp = bnxt_re_create_qp,
.create_srq = bnxt_re_create_srq,
.create_user_ah = bnxt_re_create_ah,
.dealloc_driver = bnxt_re_dealloc_driver,
.dealloc_pd = bnxt_re_dealloc_pd,
.dealloc_ucontext = bnxt_re_dealloc_ucontext,
.del_gid = bnxt_re_del_gid,
......@@ -744,18 +587,7 @@ static int bnxt_re_register_ib(struct bnxt_re_dev *rdev)
return ib_register_device(ibdev, "bnxt_re%d", &rdev->en_dev->pdev->dev);
}
static void bnxt_re_dev_remove(struct bnxt_re_dev *rdev)
{
dev_put(rdev->netdev);
rdev->netdev = NULL;
mutex_lock(&bnxt_re_dev_lock);
list_del_rcu(&rdev->list);
mutex_unlock(&bnxt_re_dev_lock);
synchronize_rcu();
}
static struct bnxt_re_dev *bnxt_re_dev_add(struct net_device *netdev,
static struct bnxt_re_dev *bnxt_re_dev_add(struct bnxt_aux_priv *aux_priv,
struct bnxt_en_dev *en_dev)
{
struct bnxt_re_dev *rdev;
......@@ -768,8 +600,8 @@ static struct bnxt_re_dev *bnxt_re_dev_add(struct net_device *netdev,
return NULL;
}
/* Default values */
rdev->netdev = netdev;
dev_hold(rdev->netdev);
rdev->nb.notifier_call = NULL;
rdev->netdev = en_dev->net;
rdev->en_dev = en_dev;
rdev->id = rdev->en_dev->pdev->devfn;
INIT_LIST_HEAD(&rdev->qp_list);
......@@ -784,9 +616,6 @@ static struct bnxt_re_dev *bnxt_re_dev_add(struct net_device *netdev,
rdev->cosq[0] = 0xFFFF;
rdev->cosq[1] = 0xFFFF;
mutex_lock(&bnxt_re_dev_lock);
list_add_tail_rcu(&rdev->list, &bnxt_re_dev_list);
mutex_unlock(&bnxt_re_dev_lock);
return rdev;
}
......@@ -930,7 +759,7 @@ static u32 bnxt_re_get_nqdb_offset(struct bnxt_re_dev *rdev, u16 indx)
return bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx) ?
(rdev->is_virtfn ? BNXT_RE_GEN_P5_VF_NQ_DB :
BNXT_RE_GEN_P5_PF_NQ_DB) :
rdev->msix_entries[indx].db_offset;
rdev->en_dev->msix_entries[indx].db_offset;
}
static void bnxt_re_cleanup_res(struct bnxt_re_dev *rdev)
......@@ -955,7 +784,7 @@ static int bnxt_re_init_res(struct bnxt_re_dev *rdev)
for (i = 1; i < rdev->num_msix ; i++) {
db_offt = bnxt_re_get_nqdb_offset(rdev, i);
rc = bnxt_qplib_enable_nq(rdev->en_dev->pdev, &rdev->nq[i - 1],
i - 1, rdev->msix_entries[i].vector,
i - 1, rdev->en_dev->msix_entries[i].vector,
db_offt, &bnxt_re_cqn_handler,
&bnxt_re_srqn_handler);
if (rc) {
......@@ -1042,7 +871,7 @@ static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev)
rattr.type = type;
rattr.mode = RING_ALLOC_REQ_INT_MODE_MSIX;
rattr.depth = BNXT_QPLIB_NQE_MAX_CNT - 1;
rattr.lrid = rdev->msix_entries[i + 1].ring_idx;
rattr.lrid = rdev->en_dev->msix_entries[i + 1].ring_idx;
rc = bnxt_re_net_ring_alloc(rdev, &rattr, &nq->ring_id);
if (rc) {
ibdev_err(&rdev->ibdev,
......@@ -1095,7 +924,6 @@ static int bnxt_re_query_hwrm_pri2cos(struct bnxt_re_dev *rdev, u8 dir,
u64 *cid_map)
{
struct hwrm_queue_pri2cos_qcfg_input req = {0};
struct bnxt *bp = netdev_priv(rdev->netdev);
struct hwrm_queue_pri2cos_qcfg_output resp;
struct bnxt_en_dev *en_dev = rdev->en_dev;
struct bnxt_fw_msg fw_msg;
......@@ -1112,11 +940,11 @@ static int bnxt_re_query_hwrm_pri2cos(struct bnxt_re_dev *rdev, u8 dir,
flags |= (dir & 0x01);
flags |= HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_IVLAN;
req.flags = cpu_to_le32(flags);
req.port_id = bp->pf.port_id;
req.port_id = en_dev->pf_port_id;
bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
rc = bnxt_send_msg(en_dev, &fw_msg);
if (rc)
return rc;
......@@ -1299,7 +1127,7 @@ static void bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev)
req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
rc = bnxt_send_msg(en_dev, &fw_msg);
if (rc) {
ibdev_err(&rdev->ibdev, "Failed to query HW version, rc = 0x%x",
rc);
......@@ -1323,7 +1151,7 @@ static int bnxt_re_ib_init(struct bnxt_re_dev *rdev)
pr_err("Failed to register with IB: %#x\n", rc);
return rc;
}
dev_info(rdev_to_dev(rdev), "Device registered successfully");
dev_info(rdev_to_dev(rdev), "Device registered with IB successfully");
ib_get_eth_speed(&rdev->ibdev, 1, &rdev->active_speed,
&rdev->active_width);
set_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &rdev->flags);
......@@ -1362,20 +1190,12 @@ static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev)
bnxt_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id, type);
bnxt_qplib_free_rcfw_channel(&rdev->rcfw);
}
if (test_and_clear_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags)) {
rc = bnxt_re_free_msix(rdev);
if (rc)
ibdev_warn(&rdev->ibdev,
"Failed to free MSI-X vectors: %#x", rc);
}
if (test_and_clear_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags))
rdev->num_msix = 0;
bnxt_re_destroy_chip_ctx(rdev);
if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags)) {
rc = bnxt_re_unregister_netdev(rdev);
if (rc)
ibdev_warn(&rdev->ibdev,
"Failed to unregister with netdev: %#x", rc);
}
if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags))
bnxt_unregister_dev(rdev->en_dev);
}
/* worker thread for polling periodic events. Now used for QoS programming*/
......@@ -1416,13 +1236,15 @@ static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 wqe_mode)
/* Check whether VF or PF */
bnxt_re_get_sriov_func_type(rdev);
rc = bnxt_re_request_msix(rdev);
if (rc) {
if (!rdev->en_dev->ulp_tbl->msix_requested) {
ibdev_err(&rdev->ibdev,
"Failed to get MSI-X vectors: %#x\n", rc);
rc = -EINVAL;
goto fail;
}
ibdev_dbg(&rdev->ibdev, "Got %d MSI-X vectors\n",
rdev->en_dev->ulp_tbl->msix_requested);
rdev->num_msix = rdev->en_dev->ulp_tbl->msix_requested;
set_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags);
bnxt_re_query_hwrm_intf_version(rdev);
......@@ -1446,14 +1268,14 @@ static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 wqe_mode)
rattr.type = type;
rattr.mode = RING_ALLOC_REQ_INT_MODE_MSIX;
rattr.depth = BNXT_QPLIB_CREQE_MAX_CNT - 1;
rattr.lrid = rdev->msix_entries[BNXT_RE_AEQ_IDX].ring_idx;
rattr.lrid = rdev->en_dev->msix_entries[BNXT_RE_AEQ_IDX].ring_idx;
rc = bnxt_re_net_ring_alloc(rdev, &rattr, &creq->ring_id);
if (rc) {
ibdev_err(&rdev->ibdev, "Failed to allocate CREQ: %#x\n", rc);
goto free_rcfw;
}
db_offt = bnxt_re_get_nqdb_offset(rdev, BNXT_RE_AEQ_IDX);
vid = rdev->msix_entries[BNXT_RE_AEQ_IDX].vector;
vid = rdev->en_dev->msix_entries[BNXT_RE_AEQ_IDX].vector;
rc = bnxt_qplib_enable_rcfw_channel(&rdev->rcfw,
vid, db_offt, rdev->is_virtfn,
&bnxt_re_aeq_handler);
......@@ -1521,6 +1343,11 @@ static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 wqe_mode)
INIT_DELAYED_WORK(&rdev->worker, bnxt_re_worker);
set_bit(BNXT_RE_FLAG_QOS_WORK_REG, &rdev->flags);
schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000));
/*
* Use the total VF count since the actual VF count may not be
* available at this point.
*/
bnxt_re_vf_res_config(rdev);
}
return 0;
......@@ -1541,135 +1368,43 @@ static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 wqe_mode)
return rc;
}
static void bnxt_re_dev_unreg(struct bnxt_re_dev *rdev)
{
struct net_device *netdev = rdev->netdev;
bnxt_re_dev_remove(rdev);
if (netdev)
dev_put(netdev);
}
static int bnxt_re_dev_reg(struct bnxt_re_dev **rdev, struct net_device *netdev)
static int bnxt_re_add_device(struct auxiliary_device *adev, u8 wqe_mode)
{
struct bnxt_aux_priv *aux_priv =
container_of(adev, struct bnxt_aux_priv, aux_dev);
struct bnxt_en_dev *en_dev;
struct bnxt_re_dev *rdev;
int rc = 0;
if (!is_bnxt_re_dev(netdev))
return -ENODEV;
/* en_dev should never be NULL as long as adev and aux_dev are valid. */
en_dev = aux_priv->edev;
en_dev = bnxt_re_dev_probe(netdev);
if (IS_ERR(en_dev)) {
if (en_dev != ERR_PTR(-ENODEV))
ibdev_err(&(*rdev)->ibdev, "%s: Failed to probe\n",
ROCE_DRV_MODULE_NAME);
rc = PTR_ERR(en_dev);
goto exit;
}
*rdev = bnxt_re_dev_add(netdev, en_dev);
if (!*rdev) {
rdev = bnxt_re_dev_add(aux_priv, en_dev);
if (!rdev || !rdev_to_dev(rdev)) {
rc = -ENOMEM;
dev_put(netdev);
goto exit;
}
exit:
return rc;
}
static void bnxt_re_remove_device(struct bnxt_re_dev *rdev)
{
bnxt_re_dev_uninit(rdev);
pci_dev_put(rdev->en_dev->pdev);
bnxt_re_dev_unreg(rdev);
}
static int bnxt_re_add_device(struct bnxt_re_dev **rdev,
struct net_device *netdev, u8 wqe_mode)
{
int rc;
rc = bnxt_re_dev_reg(rdev, netdev);
if (rc == -ENODEV)
return rc;
if (rc) {
pr_err("Failed to register with the device %s: %#x\n",
netdev->name, rc);
return rc;
}
rc = bnxt_re_dev_init(rdev, wqe_mode);
if (rc)
goto re_dev_dealloc;
pci_dev_get((*rdev)->en_dev->pdev);
rc = bnxt_re_dev_init(*rdev, wqe_mode);
rc = bnxt_re_ib_init(rdev);
if (rc) {
pci_dev_put((*rdev)->en_dev->pdev);
bnxt_re_dev_unreg(*rdev);
pr_err("Failed to register with IB: %s",
aux_priv->aux_dev.name);
goto re_dev_uninit;
}
auxiliary_set_drvdata(adev, rdev);
return rc;
}
static void bnxt_re_dealloc_driver(struct ib_device *ib_dev)
{
struct bnxt_re_dev *rdev =
container_of(ib_dev, struct bnxt_re_dev, ibdev);
dev_info(rdev_to_dev(rdev), "Unregistering Device");
rtnl_lock();
bnxt_re_remove_device(rdev);
rtnl_unlock();
}
/* Handle all deferred netevents tasks */
static void bnxt_re_task(struct work_struct *work)
{
struct bnxt_re_work *re_work;
struct bnxt_re_dev *rdev;
int rc = 0;
re_work = container_of(work, struct bnxt_re_work, work);
rdev = re_work->rdev;
if (re_work->event == NETDEV_REGISTER) {
rc = bnxt_re_ib_init(rdev);
if (rc) {
ibdev_err(&rdev->ibdev,
"Failed to register with IB: %#x", rc);
rtnl_lock();
bnxt_re_remove_device(rdev);
rtnl_unlock();
goto exit;
}
goto exit;
}
if (!ib_device_try_get(&rdev->ibdev))
goto exit;
return 0;
switch (re_work->event) {
case NETDEV_UP:
bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
IB_EVENT_PORT_ACTIVE);
break;
case NETDEV_DOWN:
bnxt_re_dev_stop(rdev);
break;
case NETDEV_CHANGE:
if (!netif_carrier_ok(rdev->netdev))
bnxt_re_dev_stop(rdev);
else if (netif_carrier_ok(rdev->netdev))
bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
IB_EVENT_PORT_ACTIVE);
ib_get_eth_speed(&rdev->ibdev, 1, &rdev->active_speed,
&rdev->active_width);
break;
default:
break;
}
ib_device_put(&rdev->ibdev);
re_dev_uninit:
bnxt_re_dev_uninit(rdev);
re_dev_dealloc:
ib_dealloc_device(&rdev->ibdev);
exit:
put_device(&rdev->ibdev.dev);
kfree(re_work);
return rc;
}
/*
......@@ -1690,109 +1425,189 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier,
unsigned long event, void *ptr)
{
struct net_device *real_dev, *netdev = netdev_notifier_info_to_dev(ptr);
struct bnxt_re_work *re_work;
struct bnxt_re_dev *rdev;
int rc = 0;
bool sch_work = false;
bool release = true;
real_dev = rdma_vlan_dev_real_dev(netdev);
if (!real_dev)
real_dev = netdev;
rdev = bnxt_re_from_netdev(real_dev);
if (!rdev && event != NETDEV_REGISTER)
return NOTIFY_OK;
if (real_dev != netdev)
goto exit;
switch (event) {
case NETDEV_REGISTER:
if (rdev)
break;
rc = bnxt_re_add_device(&rdev, real_dev,
BNXT_QPLIB_WQE_MODE_STATIC);
if (!rc)
sch_work = true;
release = false;
break;
rdev = bnxt_re_from_netdev(real_dev);
if (!rdev)
return NOTIFY_DONE;
case NETDEV_UNREGISTER:
ib_unregister_device_queued(&rdev->ibdev);
break;
switch (event) {
case NETDEV_UP:
case NETDEV_DOWN:
case NETDEV_CHANGE:
bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
netif_carrier_ok(real_dev) ?
IB_EVENT_PORT_ACTIVE :
IB_EVENT_PORT_ERR);
break;
default:
sch_work = true;
break;
}
if (sch_work) {
/* Allocate for the deferred task */
re_work = kzalloc(sizeof(*re_work), GFP_KERNEL);
if (re_work) {
get_device(&rdev->ibdev.dev);
re_work->rdev = rdev;
re_work->event = event;
re_work->vlan_dev = (real_dev == netdev ?
NULL : netdev);
INIT_WORK(&re_work->work, bnxt_re_task);
queue_work(bnxt_re_wq, &re_work->work);
}
}
ib_device_put(&rdev->ibdev);
exit:
if (rdev && release)
ib_device_put(&rdev->ibdev);
return NOTIFY_DONE;
}
static struct notifier_block bnxt_re_netdev_notifier = {
.notifier_call = bnxt_re_netdev_event
};
#define BNXT_ADEV_NAME "bnxt_en"
static int __init bnxt_re_mod_init(void)
static void bnxt_re_remove(struct auxiliary_device *adev)
{
int rc = 0;
struct bnxt_re_dev *rdev = auxiliary_get_drvdata(adev);
pr_info("%s: %s", ROCE_DRV_MODULE_NAME, version);
if (!rdev)
return;
bnxt_re_wq = create_singlethread_workqueue("bnxt_re");
if (!bnxt_re_wq)
return -ENOMEM;
mutex_lock(&bnxt_re_mutex);
if (rdev->nb.notifier_call) {
unregister_netdevice_notifier(&rdev->nb);
rdev->nb.notifier_call = NULL;
} else {
/* If notifier is null, we should have already done a
* clean up before coming here.
*/
goto skip_remove;
}
ib_unregister_device(&rdev->ibdev);
ib_dealloc_device(&rdev->ibdev);
bnxt_re_dev_uninit(rdev);
skip_remove:
mutex_unlock(&bnxt_re_mutex);
}
static int bnxt_re_probe(struct auxiliary_device *adev,
const struct auxiliary_device_id *id)
{
struct bnxt_re_dev *rdev;
int rc;
INIT_LIST_HEAD(&bnxt_re_dev_list);
mutex_lock(&bnxt_re_mutex);
rc = bnxt_re_add_device(adev, BNXT_QPLIB_WQE_MODE_STATIC);
if (rc) {
mutex_unlock(&bnxt_re_mutex);
return rc;
}
rdev = auxiliary_get_drvdata(adev);
rc = register_netdevice_notifier(&bnxt_re_netdev_notifier);
rdev->nb.notifier_call = bnxt_re_netdev_event;
rc = register_netdevice_notifier(&rdev->nb);
if (rc) {
rdev->nb.notifier_call = NULL;
pr_err("%s: Cannot register to netdevice_notifier",
ROCE_DRV_MODULE_NAME);
goto err_netdev;
goto err;
}
mutex_unlock(&bnxt_re_mutex);
return 0;
err_netdev:
destroy_workqueue(bnxt_re_wq);
err:
mutex_unlock(&bnxt_re_mutex);
bnxt_re_remove(adev);
return rc;
}
static void __exit bnxt_re_mod_exit(void)
static int bnxt_re_suspend(struct auxiliary_device *adev, pm_message_t state)
{
struct bnxt_re_dev *rdev;
struct bnxt_re_dev *rdev = auxiliary_get_drvdata(adev);
unregister_netdevice_notifier(&bnxt_re_netdev_notifier);
if (bnxt_re_wq)
destroy_workqueue(bnxt_re_wq);
list_for_each_entry(rdev, &bnxt_re_dev_list, list) {
/* VF device removal should be called before the removal
* of PF device. Queue VFs unregister first, so that VFs
* shall be removed before the PF during the call of
* ib_unregister_driver.
*/
if (rdev->is_virtfn)
ib_unregister_device(&rdev->ibdev);
if (!rdev)
return 0;
mutex_lock(&bnxt_re_mutex);
/* L2 driver may invoke this callback during device error/crash or device
* reset. Current RoCE driver doesn't recover the device in case of
* error. Handle the error by dispatching fatal events to all qps
* ie. by calling bnxt_re_dev_stop and release the MSIx vectors as
* L2 driver want to modify the MSIx table.
*/
ibdev_info(&rdev->ibdev, "Handle device suspend call");
/* Check the current device state from bnxt_en_dev and move the
* device to detached state if FW_FATAL_COND is set.
* This prevents more commands to HW during clean-up,
* in case the device is already in error.
*/
if (test_bit(BNXT_STATE_FW_FATAL_COND, &rdev->en_dev->en_state))
set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags);
bnxt_re_dev_stop(rdev);
bnxt_re_stop_irq(rdev);
/* Move the device states to detached and avoid sending any more
* commands to HW
*/
set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags);
set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags);
mutex_unlock(&bnxt_re_mutex);
return 0;
}
static int bnxt_re_resume(struct auxiliary_device *adev)
{
struct bnxt_re_dev *rdev = auxiliary_get_drvdata(adev);
if (!rdev)
return 0;
mutex_lock(&bnxt_re_mutex);
/* L2 driver may invoke this callback during device recovery, resume.
* reset. Current RoCE driver doesn't recover the device in case of
* error. Handle the error by dispatching fatal events to all qps
* ie. by calling bnxt_re_dev_stop and release the MSIx vectors as
* L2 driver want to modify the MSIx table.
*/
ibdev_info(&rdev->ibdev, "Handle device resume call");
mutex_unlock(&bnxt_re_mutex);
return 0;
}
static const struct auxiliary_device_id bnxt_re_id_table[] = {
{ .name = BNXT_ADEV_NAME ".rdma", },
{},
};
MODULE_DEVICE_TABLE(auxiliary, bnxt_re_id_table);
static struct auxiliary_driver bnxt_re_driver = {
.name = "rdma",
.probe = bnxt_re_probe,
.remove = bnxt_re_remove,
.shutdown = bnxt_re_shutdown,
.suspend = bnxt_re_suspend,
.resume = bnxt_re_resume,
.id_table = bnxt_re_id_table,
};
static int __init bnxt_re_mod_init(void)
{
int rc = 0;
pr_info("%s: %s", ROCE_DRV_MODULE_NAME, version);
rc = auxiliary_driver_register(&bnxt_re_driver);
if (rc) {
pr_err("%s: Failed to register auxiliary driver\n",
ROCE_DRV_MODULE_NAME);
return rc;
}
ib_unregister_driver(RDMA_DRIVER_BNXT_RE);
return 0;
}
static void __exit bnxt_re_mod_exit(void)
{
auxiliary_driver_unregister(&bnxt_re_driver);
}
module_init(bnxt_re_mod_init);
......
drivers/net/ethernet/broadcom/Kconfig
View file @ 9ac543c0
......@@ -213,6 +213,7 @@ config BNXT
select NET_DEVLINK
select PAGE_POOL
select DIMLIB
select AUXILIARY_BUS
help
This driver supports Broadcom NetXtreme-C/E 10/25/40/50 gigabit
Ethernet cards. To compile this driver as a module, choose M here:
......
drivers/net/ethernet/broadcom/bnxt/bnxt.c
View file @ 9ac543c0
......@@ -2414,7 +2414,6 @@ static int bnxt_async_event_process(struct bnxt *bp,
}
bnxt_queue_sp_work(bp);
async_event_process_exit:
bnxt_ulp_async_events(bp, cmpl);
return 0;
}
......@@ -5538,7 +5537,7 @@ int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
#endif
if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP))
if (!vnic_id && bnxt_ulp_registered(bp->edev))
req->flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
return hwrm_req_send(bp, req);
......@@ -13181,6 +13180,8 @@ static void bnxt_remove_one(struct pci_dev *pdev)
if (BNXT_PF(bp))
bnxt_sriov_disable(bp);
bnxt_rdma_aux_device_uninit(bp);
bnxt_ptp_clear(bp);
pci_disable_pcie_error_reporting(pdev);
unregister_netdev(dev);
......@@ -13776,11 +13777,13 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
bnxt_dl_fw_reporters_create(bp);
bnxt_rdma_aux_device_init(bp);
bnxt_print_device_info(bp);
pci_save_state(pdev);
return 0;
return 0;
init_err_cleanup:
bnxt_dl_unregister(bp);
init_err_dl:
......@@ -13824,7 +13827,6 @@ static void bnxt_shutdown(struct pci_dev *pdev)
if (netif_running(dev))
dev_close(dev);
bnxt_ulp_shutdown(bp);
bnxt_clear_int_mode(bp);
pci_disable_device(pdev);
......
drivers/net/ethernet/broadcom/bnxt/bnxt.h
View file @ 9ac543c0
......@@ -24,6 +24,7 @@
#include <linux/interrupt.h>
#include <linux/rhashtable.h>
#include <linux/crash_dump.h>
#include <linux/auxiliary_bus.h>
#include <net/devlink.h>
#include <net/dst_metadata.h>
#include <net/xdp.h>
......@@ -1631,6 +1632,12 @@ struct bnxt_fw_health {
#define BNXT_FW_IF_RETRY 10
#define BNXT_FW_SLOT_RESET_RETRY 4
struct bnxt_aux_priv {
struct auxiliary_device aux_dev;
struct bnxt_en_dev *edev;
int id;
};
enum board_idx {
BCM57301,
BCM57302,
......@@ -1852,6 +1859,7 @@ struct bnxt {
#define BNXT_CHIP_P4_PLUS(bp) \
(BNXT_CHIP_P4(bp) || BNXT_CHIP_P5(bp))
struct bnxt_aux_priv *aux_priv;
struct bnxt_en_dev *edev;
struct bnxt_napi **bnapi;
......
drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c
View file @ 9ac543c0
......@@ -749,7 +749,6 @@ int bnxt_cfg_hw_sriov(struct bnxt *bp, int *num_vfs, bool reset)
*num_vfs = rc;
}
bnxt_ulp_sriov_cfg(bp, *num_vfs);
return 0;
}
......@@ -823,10 +822,8 @@ static int bnxt_sriov_enable(struct bnxt *bp, int *num_vfs)
goto err_out2;
rc = pci_enable_sriov(bp->pdev, *num_vfs);
if (rc) {
bnxt_ulp_sriov_cfg(bp, 0);
if (rc)
goto err_out2;
}
return 0;
......@@ -872,8 +869,6 @@ void bnxt_sriov_disable(struct bnxt *bp)
rtnl_lock();
bnxt_restore_pf_fw_resources(bp);
rtnl_unlock();
bnxt_ulp_sriov_cfg(bp, 0);
}
int bnxt_sriov_configure(struct pci_dev *pdev, int num_vfs)
......
drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c
View file @ 9ac543c0
......@@ -19,89 +19,26 @@
#include <linux/irq.h>
#include <asm/byteorder.h>
#include <linux/bitmap.h>
#include <linux/auxiliary_bus.h>
#include "bnxt_hsi.h"
#include "bnxt.h"
#include "bnxt_hwrm.h"
#include "bnxt_ulp.h"
static int bnxt_register_dev(struct bnxt_en_dev *edev, unsigned int ulp_id,
struct bnxt_ulp_ops *ulp_ops, void *handle)
{
struct net_device *dev = edev->net;
struct bnxt *bp = netdev_priv(dev);
struct bnxt_ulp *ulp;
ASSERT_RTNL();
if (ulp_id >= BNXT_MAX_ULP)
return -EINVAL;
ulp = &edev->ulp_tbl[ulp_id];
if (rcu_access_pointer(ulp->ulp_ops)) {
netdev_err(bp->dev, "ulp id %d already registered\n", ulp_id);
return -EBUSY;
}
if (ulp_id == BNXT_ROCE_ULP) {
unsigned int max_stat_ctxs;
max_stat_ctxs = bnxt_get_max_func_stat_ctxs(bp);
if (max_stat_ctxs <= BNXT_MIN_ROCE_STAT_CTXS ||
bp->cp_nr_rings == max_stat_ctxs)
return -ENOMEM;
}
atomic_set(&ulp->ref_count, 0);
ulp->handle = handle;
rcu_assign_pointer(ulp->ulp_ops, ulp_ops);
if (ulp_id == BNXT_ROCE_ULP) {
if (test_bit(BNXT_STATE_OPEN, &bp->state))
bnxt_hwrm_vnic_cfg(bp, 0);
}
return 0;
}
static int bnxt_unregister_dev(struct bnxt_en_dev *edev, unsigned int ulp_id)
{
struct net_device *dev = edev->net;
struct bnxt *bp = netdev_priv(dev);
struct bnxt_ulp *ulp;
int i = 0;
ASSERT_RTNL();
if (ulp_id >= BNXT_MAX_ULP)
return -EINVAL;
ulp = &edev->ulp_tbl[ulp_id];
if (!rcu_access_pointer(ulp->ulp_ops)) {
netdev_err(bp->dev, "ulp id %d not registered\n", ulp_id);
return -EINVAL;
}
if (ulp_id == BNXT_ROCE_ULP && ulp->msix_requested)
edev->en_ops->bnxt_free_msix(edev, ulp_id);
if (ulp->max_async_event_id)
bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, true);
RCU_INIT_POINTER(ulp->ulp_ops, NULL);
synchronize_rcu();
ulp->max_async_event_id = 0;
ulp->async_events_bmap = NULL;
while (atomic_read(&ulp->ref_count) != 0 && i < 10) {
msleep(100);
i++;
}
return 0;
}
static DEFINE_IDA(bnxt_aux_dev_ids);
static void bnxt_fill_msix_vecs(struct bnxt *bp, struct bnxt_msix_entry *ent)
{
struct bnxt_en_dev *edev = bp->edev;
int num_msix, idx, i;
num_msix = edev->ulp_tbl[BNXT_ROCE_ULP].msix_requested;
idx = edev->ulp_tbl[BNXT_ROCE_ULP].msix_base;
if (!edev->ulp_tbl->msix_requested) {
netdev_warn(bp->dev, "Requested MSI-X vectors insufficient\n");
return;
}
num_msix = edev->ulp_tbl->msix_requested;
idx = edev->ulp_tbl->msix_base;
for (i = 0; i < num_msix; i++) {
ent[i].vector = bp->irq_tbl[idx + i].vector;
ent[i].ring_idx = idx + i;
......@@ -115,125 +52,95 @@ static void bnxt_fill_msix_vecs(struct bnxt *bp, struct bnxt_msix_entry *ent)
}
}
static int bnxt_req_msix_vecs(struct bnxt_en_dev *edev, unsigned int ulp_id,
struct bnxt_msix_entry *ent, int num_msix)
int bnxt_register_dev(struct bnxt_en_dev *edev,
struct bnxt_ulp_ops *ulp_ops,
void *handle)
{
struct net_device *dev = edev->net;
struct bnxt *bp = netdev_priv(dev);
struct bnxt_hw_resc *hw_resc;
int max_idx, max_cp_rings;
int avail_msix, idx;
int total_vecs;
int rc = 0;
ASSERT_RTNL();
if (ulp_id != BNXT_ROCE_ULP)
return -EINVAL;
if (!(bp->flags & BNXT_FLAG_USING_MSIX))
return -ENODEV;
unsigned int max_stat_ctxs;
struct bnxt_ulp *ulp;
if (edev->ulp_tbl[ulp_id].msix_requested)
return -EAGAIN;
max_stat_ctxs = bnxt_get_max_func_stat_ctxs(bp);
if (max_stat_ctxs <= BNXT_MIN_ROCE_STAT_CTXS ||
bp->cp_nr_rings == max_stat_ctxs)
return -ENOMEM;
max_cp_rings = bnxt_get_max_func_cp_rings(bp);
avail_msix = bnxt_get_avail_msix(bp, num_msix);
if (!avail_msix)
ulp = edev->ulp_tbl;
if (!ulp)
return -ENOMEM;
if (avail_msix > num_msix)
avail_msix = num_msix;
if (BNXT_NEW_RM(bp)) {
idx = bp->cp_nr_rings;
} else {
max_idx = min_t(int, bp->total_irqs, max_cp_rings);
idx = max_idx - avail_msix;
}
edev->ulp_tbl[ulp_id].msix_base = idx;
edev->ulp_tbl[ulp_id].msix_requested = avail_msix;
hw_resc = &bp->hw_resc;
total_vecs = idx + avail_msix;
if (bp->total_irqs < total_vecs ||
(BNXT_NEW_RM(bp) && hw_resc->resv_irqs < total_vecs)) {
if (netif_running(dev)) {
bnxt_close_nic(bp, true, false);
rc = bnxt_open_nic(bp, true, false);
} else {
rc = bnxt_reserve_rings(bp, true);
}
}
if (rc) {
edev->ulp_tbl[ulp_id].msix_requested = 0;
return -EAGAIN;
}
if (BNXT_NEW_RM(bp)) {
int resv_msix;
ulp->handle = handle;
rcu_assign_pointer(ulp->ulp_ops, ulp_ops);
if (test_bit(BNXT_STATE_OPEN, &bp->state))
bnxt_hwrm_vnic_cfg(bp, 0);
resv_msix = hw_resc->resv_irqs - bp->cp_nr_rings;
avail_msix = min_t(int, resv_msix, avail_msix);
edev->ulp_tbl[ulp_id].msix_requested = avail_msix;
}
bnxt_fill_msix_vecs(bp, ent);
bnxt_fill_msix_vecs(bp, bp->edev->msix_entries);
edev->flags |= BNXT_EN_FLAG_MSIX_REQUESTED;
return avail_msix;
return 0;
}
EXPORT_SYMBOL(bnxt_register_dev);
static int bnxt_free_msix_vecs(struct bnxt_en_dev *edev, unsigned int ulp_id)
void bnxt_unregister_dev(struct bnxt_en_dev *edev)
{
struct net_device *dev = edev->net;
struct bnxt *bp = netdev_priv(dev);
struct bnxt_ulp *ulp;
int i = 0;
ASSERT_RTNL();
if (ulp_id != BNXT_ROCE_ULP)
return -EINVAL;
ulp = edev->ulp_tbl;
if (ulp->msix_requested)
edev->flags &= ~BNXT_EN_FLAG_MSIX_REQUESTED;
if (!(edev->flags & BNXT_EN_FLAG_MSIX_REQUESTED))
return 0;
if (ulp->max_async_event_id)
bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, true);
edev->ulp_tbl[ulp_id].msix_requested = 0;
edev->flags &= ~BNXT_EN_FLAG_MSIX_REQUESTED;
if (netif_running(dev) && !(edev->flags & BNXT_EN_FLAG_ULP_STOPPED)) {
bnxt_close_nic(bp, true, false);
bnxt_open_nic(bp, true, false);
RCU_INIT_POINTER(ulp->ulp_ops, NULL);
synchronize_rcu();
ulp->max_async_event_id = 0;
ulp->async_events_bmap = NULL;
while (atomic_read(&ulp->ref_count) != 0 && i < 10) {
msleep(100);
i++;
}
return 0;
return;
}
EXPORT_SYMBOL(bnxt_unregister_dev);
int bnxt_get_ulp_msix_num(struct bnxt *bp)
{
if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) {
struct bnxt_en_dev *edev = bp->edev;
u32 roce_msix = BNXT_VF(bp) ?
BNXT_MAX_VF_ROCE_MSIX : BNXT_MAX_ROCE_MSIX;
return edev->ulp_tbl[BNXT_ROCE_ULP].msix_requested;
}
return 0;
return ((bp->flags & BNXT_FLAG_ROCE_CAP) ?
min_t(u32, roce_msix, num_online_cpus()) : 0);
}
int bnxt_get_ulp_msix_base(struct bnxt *bp)
{
if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) {
if (bnxt_ulp_registered(bp->edev)) {
struct bnxt_en_dev *edev = bp->edev;
if (edev->ulp_tbl[BNXT_ROCE_ULP].msix_requested)
return edev->ulp_tbl[BNXT_ROCE_ULP].msix_base;
if (edev->ulp_tbl->msix_requested)
return edev->ulp_tbl->msix_base;
}
return 0;
}
int bnxt_get_ulp_stat_ctxs(struct bnxt *bp)
{
if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) {
if (bnxt_ulp_registered(bp->edev)) {
struct bnxt_en_dev *edev = bp->edev;
if (edev->ulp_tbl[BNXT_ROCE_ULP].msix_requested)
if (edev->ulp_tbl->msix_requested)
return BNXT_MIN_ROCE_STAT_CTXS;
}
return 0;
}
static int bnxt_send_msg(struct bnxt_en_dev *edev, unsigned int ulp_id,
int bnxt_send_msg(struct bnxt_en_dev *edev,
struct bnxt_fw_msg *fw_msg)
{
struct net_device *dev = edev->net;
......@@ -243,7 +150,7 @@ static int bnxt_send_msg(struct bnxt_en_dev *edev, unsigned int ulp_id,
u32 resp_len;
int rc;
if (ulp_id != BNXT_ROCE_ULP && bp->fw_reset_state)
if (bp->fw_reset_state)
return -EBUSY;
rc = hwrm_req_init(bp, req, 0 /* don't care */);
......@@ -267,42 +174,36 @@ static int bnxt_send_msg(struct bnxt_en_dev *edev, unsigned int ulp_id,
hwrm_req_drop(bp, req);
return rc;
}
static void bnxt_ulp_get(struct bnxt_ulp *ulp)
{
atomic_inc(&ulp->ref_count);
}
static void bnxt_ulp_put(struct bnxt_ulp *ulp)
{
atomic_dec(&ulp->ref_count);
}
EXPORT_SYMBOL(bnxt_send_msg);
void bnxt_ulp_stop(struct bnxt *bp)
{
struct bnxt_aux_priv *aux_priv = bp->aux_priv;
struct bnxt_en_dev *edev = bp->edev;
struct bnxt_ulp_ops *ops;
int i;
if (!edev)
return;
edev->flags |= BNXT_EN_FLAG_ULP_STOPPED;
for (i = 0; i < BNXT_MAX_ULP; i++) {
struct bnxt_ulp *ulp = &edev->ulp_tbl[i];
if (aux_priv) {
struct auxiliary_device *adev;
ops = rtnl_dereference(ulp->ulp_ops);
if (!ops || !ops->ulp_stop)
continue;
ops->ulp_stop(ulp->handle);
adev = &aux_priv->aux_dev;
if (adev->dev.driver) {
struct auxiliary_driver *adrv;
pm_message_t pm = {};
adrv = to_auxiliary_drv(adev->dev.driver);
edev->en_state = bp->state;
adrv->suspend(adev, pm);
}
}
}
void bnxt_ulp_start(struct bnxt *bp, int err)
{
struct bnxt_aux_priv *aux_priv = bp->aux_priv;
struct bnxt_en_dev *edev = bp->edev;
struct bnxt_ulp_ops *ops;
int i;
if (!edev)
return;
......@@ -312,58 +213,19 @@ void bnxt_ulp_start(struct bnxt *bp, int err)
if (err)
return;
for (i = 0; i < BNXT_MAX_ULP; i++) {
struct bnxt_ulp *ulp = &edev->ulp_tbl[i];
ops = rtnl_dereference(ulp->ulp_ops);
if (!ops || !ops->ulp_start)
continue;
ops->ulp_start(ulp->handle);
}
}
void bnxt_ulp_sriov_cfg(struct bnxt *bp, int num_vfs)
{
struct bnxt_en_dev *edev = bp->edev;
struct bnxt_ulp_ops *ops;
int i;
if (!edev)
return;
if (aux_priv) {
struct auxiliary_device *adev;
for (i = 0; i < BNXT_MAX_ULP; i++) {
struct bnxt_ulp *ulp = &edev->ulp_tbl[i];
adev = &aux_priv->aux_dev;
if (adev->dev.driver) {
struct auxiliary_driver *adrv;
rcu_read_lock();
ops = rcu_dereference(ulp->ulp_ops);
if (!ops || !ops->ulp_sriov_config) {
rcu_read_unlock();
continue;
adrv = to_auxiliary_drv(adev->dev.driver);
edev->en_state = bp->state;
adrv->resume(adev);
}
bnxt_ulp_get(ulp);
rcu_read_unlock();
ops->ulp_sriov_config(ulp->handle, num_vfs);
bnxt_ulp_put(ulp);
}
}
void bnxt_ulp_shutdown(struct bnxt *bp)
{
struct bnxt_en_dev *edev = bp->edev;
struct bnxt_ulp_ops *ops;
int i;
if (!edev)
return;
for (i = 0; i < BNXT_MAX_ULP; i++) {
struct bnxt_ulp *ulp = &edev->ulp_tbl[i];
ops = rtnl_dereference(ulp->ulp_ops);
if (!ops || !ops->ulp_shutdown)
continue;
ops->ulp_shutdown(ulp->handle);
}
}
void bnxt_ulp_irq_stop(struct bnxt *bp)
......@@ -374,8 +236,8 @@ void bnxt_ulp_irq_stop(struct bnxt *bp)
if (!edev || !(edev->flags & BNXT_EN_FLAG_MSIX_REQUESTED))
return;
if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) {
struct bnxt_ulp *ulp = &edev->ulp_tbl[BNXT_ROCE_ULP];
if (bnxt_ulp_registered(bp->edev)) {
struct bnxt_ulp *ulp = edev->ulp_tbl;
if (!ulp->msix_requested)
return;
......@@ -395,8 +257,8 @@ void bnxt_ulp_irq_restart(struct bnxt *bp, int err)
if (!edev || !(edev->flags & BNXT_EN_FLAG_MSIX_REQUESTED))
return;
if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) {
struct bnxt_ulp *ulp = &edev->ulp_tbl[BNXT_ROCE_ULP];
if (bnxt_ulp_registered(bp->edev)) {
struct bnxt_ulp *ulp = edev->ulp_tbl;
struct bnxt_msix_entry *ent = NULL;
if (!ulp->msix_requested)
......@@ -418,46 +280,15 @@ void bnxt_ulp_irq_restart(struct bnxt *bp, int err)
}
}
void bnxt_ulp_async_events(struct bnxt *bp, struct hwrm_async_event_cmpl *cmpl)
{
u16 event_id = le16_to_cpu(cmpl->event_id);
struct bnxt_en_dev *edev = bp->edev;
struct bnxt_ulp_ops *ops;
int i;
if (!edev)
return;
rcu_read_lock();
for (i = 0; i < BNXT_MAX_ULP; i++) {
struct bnxt_ulp *ulp = &edev->ulp_tbl[i];
ops = rcu_dereference(ulp->ulp_ops);
if (!ops || !ops->ulp_async_notifier)
continue;
if (!ulp->async_events_bmap ||
event_id > ulp->max_async_event_id)
continue;
/* Read max_async_event_id first before testing the bitmap. */
smp_rmb();
if (test_bit(event_id, ulp->async_events_bmap))
ops->ulp_async_notifier(ulp->handle, cmpl);
}
rcu_read_unlock();
}
static int bnxt_register_async_events(struct bnxt_en_dev *edev, unsigned int ulp_id,
unsigned long *events_bmap, u16 max_id)
int bnxt_register_async_events(struct bnxt_en_dev *edev,
unsigned long *events_bmap,
u16 max_id)
{
struct net_device *dev = edev->net;
struct bnxt *bp = netdev_priv(dev);
struct bnxt_ulp *ulp;
if (ulp_id >= BNXT_MAX_ULP)
return -EINVAL;
ulp = &edev->ulp_tbl[ulp_id];
ulp = edev->ulp_tbl;
ulp->async_events_bmap = events_bmap;
/* Make sure bnxt_ulp_async_events() sees this order */
smp_wmb();
......@@ -465,38 +296,121 @@ static int bnxt_register_async_events(struct bnxt_en_dev *edev, unsigned int ulp
bnxt_hwrm_func_drv_rgtr(bp, events_bmap, max_id + 1, true);
return 0;
}
EXPORT_SYMBOL(bnxt_register_async_events);
static const struct bnxt_en_ops bnxt_en_ops_tbl = {
.bnxt_register_device = bnxt_register_dev,
.bnxt_unregister_device = bnxt_unregister_dev,
.bnxt_request_msix = bnxt_req_msix_vecs,
.bnxt_free_msix = bnxt_free_msix_vecs,
.bnxt_send_fw_msg = bnxt_send_msg,
.bnxt_register_fw_async_events = bnxt_register_async_events,
};
void bnxt_rdma_aux_device_uninit(struct bnxt *bp)
{
struct bnxt_aux_priv *aux_priv;
struct auxiliary_device *adev;
/* Skip if no auxiliary device init was done. */
if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
return;
aux_priv = bp->aux_priv;
adev = &aux_priv->aux_dev;
auxiliary_device_delete(adev);
auxiliary_device_uninit(adev);
}
struct bnxt_en_dev *bnxt_ulp_probe(struct net_device *dev)
static void bnxt_aux_dev_release(struct device *dev)
{
struct bnxt *bp = netdev_priv(dev);
struct bnxt_en_dev *edev;
struct bnxt_aux_priv *aux_priv =
container_of(dev, struct bnxt_aux_priv, aux_dev.dev);
ida_free(&bnxt_aux_dev_ids, aux_priv->id);
kfree(aux_priv->edev->ulp_tbl);
kfree(aux_priv->edev);
kfree(aux_priv);
}
static void bnxt_set_edev_info(struct bnxt_en_dev *edev, struct bnxt *bp)
{
edev->net = bp->dev;
edev->pdev = bp->pdev;
edev->l2_db_size = bp->db_size;
edev->l2_db_size_nc = bp->db_size;
edev = bp->edev;
if (!edev) {
edev = kzalloc(sizeof(*edev), GFP_KERNEL);
if (!edev)
return ERR_PTR(-ENOMEM);
edev->en_ops = &bnxt_en_ops_tbl;
edev->net = dev;
edev->pdev = bp->pdev;
edev->l2_db_size = bp->db_size;
edev->l2_db_size_nc = bp->db_size;
bp->edev = edev;
}
edev->flags &= ~BNXT_EN_FLAG_ROCE_CAP;
if (bp->flags & BNXT_FLAG_ROCEV1_CAP)
edev->flags |= BNXT_EN_FLAG_ROCEV1_CAP;
if (bp->flags & BNXT_FLAG_ROCEV2_CAP)
edev->flags |= BNXT_EN_FLAG_ROCEV2_CAP;
return bp->edev;
if (bp->flags & BNXT_FLAG_VF)
edev->flags |= BNXT_EN_FLAG_VF;
edev->chip_num = bp->chip_num;
edev->hw_ring_stats_size = bp->hw_ring_stats_size;
edev->pf_port_id = bp->pf.port_id;
edev->en_state = bp->state;
edev->ulp_tbl->msix_requested = bnxt_get_ulp_msix_num(bp);
}
void bnxt_rdma_aux_device_init(struct bnxt *bp)
{
struct auxiliary_device *aux_dev;
struct bnxt_aux_priv *aux_priv;
struct bnxt_en_dev *edev;
struct bnxt_ulp *ulp;
int rc;
if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
return;
bp->aux_priv = kzalloc(sizeof(*bp->aux_priv), GFP_KERNEL);
if (!bp->aux_priv)
goto exit;
bp->aux_priv->id = ida_alloc(&bnxt_aux_dev_ids, GFP_KERNEL);
if (bp->aux_priv->id < 0) {
netdev_warn(bp->dev,
"ida alloc failed for ROCE auxiliary device\n");
kfree(bp->aux_priv);
goto exit;
}
aux_priv = bp->aux_priv;
aux_dev = &aux_priv->aux_dev;
aux_dev->id = aux_priv->id;
aux_dev->name = "rdma";
aux_dev->dev.parent = &bp->pdev->dev;
aux_dev->dev.release = bnxt_aux_dev_release;
rc = auxiliary_device_init(aux_dev);
if (rc) {
ida_free(&bnxt_aux_dev_ids, bp->aux_priv->id);
kfree(bp->aux_priv);
goto exit;
}
/* From this point, all cleanup will happen via the .release callback &
* any error unwinding will need to include a call to
* auxiliary_device_uninit.
*/
edev = kzalloc(sizeof(*edev), GFP_KERNEL);
if (!edev)
goto aux_dev_uninit;
ulp = kzalloc(sizeof(*ulp), GFP_KERNEL);
if (!ulp)
goto aux_dev_uninit;
edev->ulp_tbl = ulp;
aux_priv->edev = edev;
bp->edev = edev;
bnxt_set_edev_info(edev, bp);
rc = auxiliary_device_add(aux_dev);
if (rc) {
netdev_warn(bp->dev,
"Failed to add auxiliary device for ROCE\n");
goto aux_dev_uninit;
}
return;
aux_dev_uninit:
auxiliary_device_uninit(aux_dev);
exit:
bp->flags &= ~BNXT_FLAG_ROCE_CAP;
}
EXPORT_SYMBOL(bnxt_ulp_probe);
drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.h
View file @ 9ac543c0
......@@ -15,6 +15,8 @@
#define BNXT_MIN_ROCE_CP_RINGS 2
#define BNXT_MIN_ROCE_STAT_CTXS 1
#define BNXT_MAX_ROCE_MSIX 9
#define BNXT_MAX_VF_ROCE_MSIX 2
struct hwrm_async_event_cmpl;
struct bnxt;
......@@ -26,12 +28,6 @@ struct bnxt_msix_entry {
};
struct bnxt_ulp_ops {
/* async_notifier() cannot sleep (in BH context) */
void (*ulp_async_notifier)(void *, struct hwrm_async_event_cmpl *);
void (*ulp_stop)(void *);
void (*ulp_start)(void *);
void (*ulp_sriov_config)(void *, int);
void (*ulp_shutdown)(void *);
void (*ulp_irq_stop)(void *);
void (*ulp_irq_restart)(void *, struct bnxt_msix_entry *);
};
......@@ -57,6 +53,7 @@ struct bnxt_ulp {
struct bnxt_en_dev {
struct net_device *net;
struct pci_dev *pdev;
struct bnxt_msix_entry msix_entries[BNXT_MAX_ROCE_MSIX];
u32 flags;
#define BNXT_EN_FLAG_ROCEV1_CAP 0x1
#define BNXT_EN_FLAG_ROCEV2_CAP 0x2
......@@ -64,8 +61,10 @@ struct bnxt_en_dev {
BNXT_EN_FLAG_ROCEV2_CAP)
#define BNXT_EN_FLAG_MSIX_REQUESTED 0x4
#define BNXT_EN_FLAG_ULP_STOPPED 0x8
const struct bnxt_en_ops *en_ops;
struct bnxt_ulp ulp_tbl[BNXT_MAX_ULP];
#define BNXT_EN_FLAG_VF 0x10
#define BNXT_EN_VF(edev) ((edev)->flags & BNXT_EN_FLAG_VF)
struct bnxt_ulp *ulp_tbl;
int l2_db_size; /* Doorbell BAR size in
* bytes mapped by L2
* driver.
......@@ -74,24 +73,19 @@ struct bnxt_en_dev {
* bytes mapped as non-
* cacheable.
*/
u16 chip_num;
u16 hw_ring_stats_size;
u16 pf_port_id;
unsigned long en_state; /* Could be checked in
* RoCE driver suspend
* mode only. Will be
* updated in resume.
*/
};
struct bnxt_en_ops {
int (*bnxt_register_device)(struct bnxt_en_dev *, unsigned int,
struct bnxt_ulp_ops *, void *);
int (*bnxt_unregister_device)(struct bnxt_en_dev *, unsigned int);
int (*bnxt_request_msix)(struct bnxt_en_dev *, unsigned int,
struct bnxt_msix_entry *, int);
int (*bnxt_free_msix)(struct bnxt_en_dev *, unsigned int);
int (*bnxt_send_fw_msg)(struct bnxt_en_dev *, unsigned int,
struct bnxt_fw_msg *);
int (*bnxt_register_fw_async_events)(struct bnxt_en_dev *, unsigned int,
unsigned long *, u16);
};
static inline bool bnxt_ulp_registered(struct bnxt_en_dev *edev, int ulp_id)
static inline bool bnxt_ulp_registered(struct bnxt_en_dev *edev)
{
if (edev && rcu_access_pointer(edev->ulp_tbl[ulp_id].ulp_ops))
if (edev && edev->ulp_tbl)
return true;
return false;
}
......@@ -102,10 +96,15 @@ int bnxt_get_ulp_stat_ctxs(struct bnxt *bp);
void bnxt_ulp_stop(struct bnxt *bp);
void bnxt_ulp_start(struct bnxt *bp, int err);
void bnxt_ulp_sriov_cfg(struct bnxt *bp, int num_vfs);
void bnxt_ulp_shutdown(struct bnxt *bp);
void bnxt_ulp_irq_stop(struct bnxt *bp);
void bnxt_ulp_irq_restart(struct bnxt *bp, int err);
void bnxt_ulp_async_events(struct bnxt *bp, struct hwrm_async_event_cmpl *cmpl);
struct bnxt_en_dev *bnxt_ulp_probe(struct net_device *dev);
void bnxt_rdma_aux_device_uninit(struct bnxt *bp);
void bnxt_rdma_aux_device_init(struct bnxt *bp);
int bnxt_register_dev(struct bnxt_en_dev *edev, struct bnxt_ulp_ops *ulp_ops,
void *handle);
void bnxt_unregister_dev(struct bnxt_en_dev *edev);
int bnxt_send_msg(struct bnxt_en_dev *edev, struct bnxt_fw_msg *fw_msg);
int bnxt_register_async_events(struct bnxt_en_dev *edev,
unsigned long *events_bmap, u16 max_id);
#endif
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7