Commit 63cbc552 authored by Feras Daoud's avatar Feras Daoud Committed by Saeed Mahameed

net/mlx5: Handle SW reset of FW in error flow

New mlx5 adapters allow the driver to reset the FW in the event of an
error, this action called "SW Reset". When an SW reset is issued on any
PF all PFs enter reset state which is a recoverable condition. The
existing recovery flow was designed to allow the recovery of a VF after
a PF driver reload. This patch adds the sw reset to the NIC states
as a preparation for sw reset handling.

When a software reset is issued the following occurs:
1. The NIC interface mode is set to 7 while the reset is in progress.
2. Once the reset completes the NIC interface mode is set to 1.
Signed-off-by: default avatarFeras Daoud <ferasda@mellanox.com>
Signed-off-by: default avatarMoshe Shemesh <moshe@mellanox.com>
Signed-off-by: default avatarDaniel Jurgens <danielj@mellanox.com>
Reviewed-by: default avatarAlex Vesker <valex@mellanox.com>
Signed-off-by: default avatarSaeed Mahameed <saeedm@mellanox.com>
parent 8b9d8baa
...@@ -64,7 +64,7 @@ static int mlx5e_test_health_info(struct mlx5e_priv *priv) ...@@ -64,7 +64,7 @@ static int mlx5e_test_health_info(struct mlx5e_priv *priv)
{ {
struct mlx5_core_health *health = &priv->mdev->priv.health; struct mlx5_core_health *health = &priv->mdev->priv.health;
return health->sick ? 1 : 0; return health->fatal_error ? 1 : 0;
} }
static int mlx5e_test_link_state(struct mlx5e_priv *priv) static int mlx5e_test_link_state(struct mlx5e_priv *priv)
......
...@@ -62,12 +62,18 @@ enum { ...@@ -62,12 +62,18 @@ enum {
enum { enum {
MLX5_DROP_NEW_HEALTH_WORK, MLX5_DROP_NEW_HEALTH_WORK,
MLX5_DROP_NEW_RECOVERY_WORK, };
enum {
MLX5_SENSOR_NO_ERR = 0,
MLX5_SENSOR_PCI_COMM_ERR = 1,
MLX5_SENSOR_NIC_DISABLED = 2,
MLX5_SENSOR_NIC_SW_RESET = 3,
}; };
u8 mlx5_get_nic_state(struct mlx5_core_dev *dev) u8 mlx5_get_nic_state(struct mlx5_core_dev *dev)
{ {
return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 3; return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 7;
} }
void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state) void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state)
...@@ -80,18 +86,25 @@ void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state) ...@@ -80,18 +86,25 @@ void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state)
&dev->iseg->cmdq_addr_l_sz); &dev->iseg->cmdq_addr_l_sz);
} }
static int in_fatal(struct mlx5_core_dev *dev) static bool sensor_pci_not_working(struct mlx5_core_dev *dev)
{ {
struct mlx5_core_health *health = &dev->priv.health; struct mlx5_core_health *health = &dev->priv.health;
struct health_buffer __iomem *h = health->health; struct health_buffer __iomem *h = health->health;
if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED) /* Offline PCI reads return 0xffffffff */
return 1; return (ioread32be(&h->fw_ver) == 0xffffffff);
}
if (ioread32be(&h->fw_ver) == 0xffffffff) static u32 check_fatal_sensors(struct mlx5_core_dev *dev)
return 1; {
if (sensor_pci_not_working(dev))
return MLX5_SENSOR_PCI_COMM_ERR;
if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED)
return MLX5_SENSOR_NIC_DISABLED;
if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_SW_RESET)
return MLX5_SENSOR_NIC_SW_RESET;
return 0; return MLX5_SENSOR_NO_ERR;
} }
void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force) void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force)
...@@ -101,7 +114,8 @@ void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force) ...@@ -101,7 +114,8 @@ void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force)
goto unlock; goto unlock;
mlx5_core_err(dev, "start\n"); mlx5_core_err(dev, "start\n");
if (pci_channel_offline(dev->pdev) || in_fatal(dev) || force) { if (pci_channel_offline(dev->pdev) ||
dev->priv.health.fatal_error != MLX5_SENSOR_NO_ERR || force) {
dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
mlx5_cmd_flush(dev); mlx5_cmd_flush(dev);
} }
...@@ -137,38 +151,14 @@ static void mlx5_handle_bad_state(struct mlx5_core_dev *dev) ...@@ -137,38 +151,14 @@ static void mlx5_handle_bad_state(struct mlx5_core_dev *dev)
mlx5_disable_device(dev); mlx5_disable_device(dev);
} }
static void health_recover(struct work_struct *work)
{
struct mlx5_core_health *health;
struct delayed_work *dwork;
struct mlx5_core_dev *dev;
struct mlx5_priv *priv;
u8 nic_state;
dwork = container_of(work, struct delayed_work, work);
health = container_of(dwork, struct mlx5_core_health, recover_work);
priv = container_of(health, struct mlx5_priv, health);
dev = container_of(priv, struct mlx5_core_dev, priv);
nic_state = mlx5_get_nic_state(dev);
if (nic_state == MLX5_NIC_IFC_INVALID) {
mlx5_core_err(dev, "health recovery flow aborted since the nic state is invalid\n");
return;
}
mlx5_core_err(dev, "starting health recovery flow\n");
mlx5_recover_device(dev);
}
/* How much time to wait until health resetting the driver (in msecs) */ /* How much time to wait until health resetting the driver (in msecs) */
#define MLX5_RECOVERY_DELAY_MSECS 60000 #define MLX5_RECOVERY_WAIT_MSECS 60000
static void health_care(struct work_struct *work) static void health_care(struct work_struct *work)
{ {
unsigned long recover_delay = msecs_to_jiffies(MLX5_RECOVERY_DELAY_MSECS);
struct mlx5_core_health *health; struct mlx5_core_health *health;
struct mlx5_core_dev *dev; struct mlx5_core_dev *dev;
struct mlx5_priv *priv; struct mlx5_priv *priv;
unsigned long flags; unsigned long end;
health = container_of(work, struct mlx5_core_health, work); health = container_of(work, struct mlx5_core_health, work);
priv = container_of(health, struct mlx5_priv, health); priv = container_of(health, struct mlx5_priv, health);
...@@ -176,13 +166,18 @@ static void health_care(struct work_struct *work) ...@@ -176,13 +166,18 @@ static void health_care(struct work_struct *work)
mlx5_core_warn(dev, "handling bad device here\n"); mlx5_core_warn(dev, "handling bad device here\n");
mlx5_handle_bad_state(dev); mlx5_handle_bad_state(dev);
spin_lock_irqsave(&health->wq_lock, flags); end = jiffies + msecs_to_jiffies(MLX5_RECOVERY_WAIT_MSECS);
if (!test_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags)) while (sensor_pci_not_working(dev)) {
schedule_delayed_work(&health->recover_work, recover_delay); if (time_after(jiffies, end)) {
else
mlx5_core_err(dev, mlx5_core_err(dev,
"new health works are not permitted at this stage\n"); "health recovery flow aborted, PCI reads still not working\n");
spin_unlock_irqrestore(&health->wq_lock, flags); return;
}
msleep(100);
}
mlx5_core_err(dev, "starting health recovery flow\n");
mlx5_recover_device(dev);
} }
static const char *hsynd_str(u8 synd) static const char *hsynd_str(u8 synd)
...@@ -274,6 +269,7 @@ static void poll_health(struct timer_list *t) ...@@ -274,6 +269,7 @@ static void poll_health(struct timer_list *t)
{ {
struct mlx5_core_dev *dev = from_timer(dev, t, priv.health.timer); struct mlx5_core_dev *dev = from_timer(dev, t, priv.health.timer);
struct mlx5_core_health *health = &dev->priv.health; struct mlx5_core_health *health = &dev->priv.health;
u32 fatal_error;
u32 count; u32 count;
if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
...@@ -291,8 +287,11 @@ static void poll_health(struct timer_list *t) ...@@ -291,8 +287,11 @@ static void poll_health(struct timer_list *t)
print_health_info(dev); print_health_info(dev);
} }
if (in_fatal(dev) && !health->sick) { fatal_error = check_fatal_sensors(dev);
health->sick = true;
if (fatal_error && !health->fatal_error) {
mlx5_core_err(dev, "Fatal error %u detected\n", fatal_error);
dev->priv.health.fatal_error = fatal_error;
print_health_info(dev); print_health_info(dev);
mlx5_trigger_health_work(dev); mlx5_trigger_health_work(dev);
} }
...@@ -306,9 +305,8 @@ void mlx5_start_health_poll(struct mlx5_core_dev *dev) ...@@ -306,9 +305,8 @@ void mlx5_start_health_poll(struct mlx5_core_dev *dev)
struct mlx5_core_health *health = &dev->priv.health; struct mlx5_core_health *health = &dev->priv.health;
timer_setup(&health->timer, poll_health, 0); timer_setup(&health->timer, poll_health, 0);
health->sick = 0; health->fatal_error = MLX5_SENSOR_NO_ERR;
clear_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags); clear_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
clear_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags);
health->health = &dev->iseg->health; health->health = &dev->iseg->health;
health->health_counter = &dev->iseg->health_counter; health->health_counter = &dev->iseg->health_counter;
...@@ -324,7 +322,6 @@ void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health) ...@@ -324,7 +322,6 @@ void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health)
if (disable_health) { if (disable_health) {
spin_lock_irqsave(&health->wq_lock, flags); spin_lock_irqsave(&health->wq_lock, flags);
set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags); set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags);
spin_unlock_irqrestore(&health->wq_lock, flags); spin_unlock_irqrestore(&health->wq_lock, flags);
} }
...@@ -338,23 +335,10 @@ void mlx5_drain_health_wq(struct mlx5_core_dev *dev) ...@@ -338,23 +335,10 @@ void mlx5_drain_health_wq(struct mlx5_core_dev *dev)
spin_lock_irqsave(&health->wq_lock, flags); spin_lock_irqsave(&health->wq_lock, flags);
set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags); set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags);
spin_unlock_irqrestore(&health->wq_lock, flags); spin_unlock_irqrestore(&health->wq_lock, flags);
cancel_delayed_work_sync(&health->recover_work);
cancel_work_sync(&health->work); cancel_work_sync(&health->work);
} }
void mlx5_drain_health_recovery(struct mlx5_core_dev *dev)
{
struct mlx5_core_health *health = &dev->priv.health;
unsigned long flags;
spin_lock_irqsave(&health->wq_lock, flags);
set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags);
spin_unlock_irqrestore(&health->wq_lock, flags);
cancel_delayed_work_sync(&dev->priv.health.recover_work);
}
void mlx5_health_flush(struct mlx5_core_dev *dev) void mlx5_health_flush(struct mlx5_core_dev *dev)
{ {
struct mlx5_core_health *health = &dev->priv.health; struct mlx5_core_health *health = &dev->priv.health;
...@@ -387,7 +371,6 @@ int mlx5_health_init(struct mlx5_core_dev *dev) ...@@ -387,7 +371,6 @@ int mlx5_health_init(struct mlx5_core_dev *dev)
return -ENOMEM; return -ENOMEM;
spin_lock_init(&health->wq_lock); spin_lock_init(&health->wq_lock);
INIT_WORK(&health->work, health_care); INIT_WORK(&health->work, health_care);
INIT_DELAYED_WORK(&health->recover_work, health_recover);
return 0; return 0;
} }
...@@ -1191,7 +1191,7 @@ static int mlx5_unload_one(struct mlx5_core_dev *dev, bool cleanup) ...@@ -1191,7 +1191,7 @@ static int mlx5_unload_one(struct mlx5_core_dev *dev, bool cleanup)
int err = 0; int err = 0;
if (cleanup) if (cleanup)
mlx5_drain_health_recovery(dev); mlx5_drain_health_wq(dev);
mutex_lock(&dev->intf_state_mutex); mutex_lock(&dev->intf_state_mutex);
if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) { if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) {
......
...@@ -214,7 +214,7 @@ enum { ...@@ -214,7 +214,7 @@ enum {
MLX5_NIC_IFC_FULL = 0, MLX5_NIC_IFC_FULL = 0,
MLX5_NIC_IFC_DISABLED = 1, MLX5_NIC_IFC_DISABLED = 1,
MLX5_NIC_IFC_NO_DRAM_NIC = 2, MLX5_NIC_IFC_NO_DRAM_NIC = 2,
MLX5_NIC_IFC_INVALID = 3 MLX5_NIC_IFC_SW_RESET = 7
}; };
u8 mlx5_get_nic_state(struct mlx5_core_dev *dev); u8 mlx5_get_nic_state(struct mlx5_core_dev *dev);
......
...@@ -435,6 +435,7 @@ struct mlx5_core_health { ...@@ -435,6 +435,7 @@ struct mlx5_core_health {
u32 prev; u32 prev;
int miss_counter; int miss_counter;
bool sick; bool sick;
u32 fatal_error;
u32 crdump_size; u32 crdump_size;
/* wq spinlock to synchronize draining */ /* wq spinlock to synchronize draining */
spinlock_t wq_lock; spinlock_t wq_lock;
...@@ -906,7 +907,6 @@ void mlx5_start_health_poll(struct mlx5_core_dev *dev); ...@@ -906,7 +907,6 @@ void mlx5_start_health_poll(struct mlx5_core_dev *dev);
void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health); void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health);
void mlx5_drain_health_wq(struct mlx5_core_dev *dev); void mlx5_drain_health_wq(struct mlx5_core_dev *dev);
void mlx5_trigger_health_work(struct mlx5_core_dev *dev); void mlx5_trigger_health_work(struct mlx5_core_dev *dev);
void mlx5_drain_health_recovery(struct mlx5_core_dev *dev);
int mlx5_buf_alloc_node(struct mlx5_core_dev *dev, int size, int mlx5_buf_alloc_node(struct mlx5_core_dev *dev, int size,
struct mlx5_frag_buf *buf, int node); struct mlx5_frag_buf *buf, int node);
int mlx5_buf_alloc(struct mlx5_core_dev *dev, int mlx5_buf_alloc(struct mlx5_core_dev *dev,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment