Commit fc19de38 authored by Zach Brown's avatar Zach Brown Committed by Andy Grover

RDS/IB: disconnect when IB devices are removed

Currently IB device removal destroys connections which are associated with the
device.  This prevents connections from being re-established when replacement
devices are added.

Instead we'll queue shutdown work on the connections as their devices are
removed.  When we see that devices are added we triger connection attempts on
all connections that don't currently have a device.

The result is that RDS sockets can resume device-independent work (bcopy, not
RDMA) across IB device removal and restoration.
Signed-off-by: default avatarZach Brown <zach.brown@oracle.com>
parent f3c6808d
...@@ -59,6 +59,27 @@ struct list_head rds_ib_devices; ...@@ -59,6 +59,27 @@ struct list_head rds_ib_devices;
DEFINE_SPINLOCK(ib_nodev_conns_lock); DEFINE_SPINLOCK(ib_nodev_conns_lock);
LIST_HEAD(ib_nodev_conns); LIST_HEAD(ib_nodev_conns);
void rds_ib_nodev_connect(void)
{
struct rds_ib_connection *ic;
spin_lock(&ib_nodev_conns_lock);
list_for_each_entry(ic, &ib_nodev_conns, ib_node)
rds_conn_connect_if_down(ic->conn);
spin_unlock(&ib_nodev_conns_lock);
}
void rds_ib_dev_shutdown(struct rds_ib_device *rds_ibdev)
{
struct rds_ib_connection *ic;
unsigned long flags;
spin_lock_irqsave(&rds_ibdev->spinlock, flags);
list_for_each_entry(ic, &rds_ibdev->conn_list, ib_node)
rds_conn_drop(ic->conn);
spin_unlock_irqrestore(&rds_ibdev->spinlock, flags);
}
/* /*
* rds_ib_destroy_mr_pool() blocks on a few things and mrs drop references * rds_ib_destroy_mr_pool() blocks on a few things and mrs drop references
* from interrupt context so we push freing off into a work struct in krdsd. * from interrupt context so we push freing off into a work struct in krdsd.
...@@ -156,6 +177,8 @@ void rds_ib_add_one(struct ib_device *device) ...@@ -156,6 +177,8 @@ void rds_ib_add_one(struct ib_device *device)
ib_set_client_data(device, &rds_ib_client, rds_ibdev); ib_set_client_data(device, &rds_ib_client, rds_ibdev);
atomic_inc(&rds_ibdev->refcount); atomic_inc(&rds_ibdev->refcount);
rds_ib_nodev_connect();
put_dev: put_dev:
rds_ib_dev_put(rds_ibdev); rds_ib_dev_put(rds_ibdev);
free_attr: free_attr:
...@@ -205,7 +228,7 @@ void rds_ib_remove_one(struct ib_device *device) ...@@ -205,7 +228,7 @@ void rds_ib_remove_one(struct ib_device *device)
if (!rds_ibdev) if (!rds_ibdev)
return; return;
rds_ib_destroy_conns(rds_ibdev); rds_ib_dev_shutdown(rds_ibdev);
/* /*
* prevent future connection attempts from getting a reference to this * prevent future connection attempts from getting a reference to this
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment