IB/core: Free GID table entry during GID deletion

If we already hold the table->lock when doing the kref_put it means we are
in a context where it is safe to do the deletion synchronously, with no
need for the work queue.

This helps to eliminate issues when GID change is requested as part of MAC
address change or bonding event change where expectation is to replace the
GID almost immediately.

Fixes: b150c386 ("IB/core: Introduce GID entry reference counts")
Reviewed-by: Daniel Jurgens <danielj@mellanox.com>
Signed-off-by: Parav Pandit <parav@mellanox.com>
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>

drivers/infiniband/core/cache.c

@@ -206,7 +206,7 @@ static void schedule_free_gid(struct kref *kref)
 	queue_work(ib_wq, &entry->del_work);
 }
 
-static void free_gid_entry(struct ib_gid_table_entry *entry)
+static void free_gid_entry_locked(struct ib_gid_table_entry *entry)
 {
 	struct ib_device *device = entry->attr.device;
 	u8 port_num = entry->attr.port_num;
@@ -216,10 +216,10 @@ static void free_gid_entry(struct ib_gid_table_entry *entry)
 		 device->name, port_num, entry->attr.index,
 		 entry->attr.gid.raw);
 
-	mutex_lock(&table->lock);
 	if (rdma_cap_roce_gid_table(device, port_num) &&
 	    entry->state != GID_TABLE_ENTRY_INVALID)
 		device->del_gid(&entry->attr, &entry->context);
+
 	write_lock_irq(&table->rwlock);
 
 	/*
@@ -232,13 +232,20 @@ static void free_gid_entry(struct ib_gid_table_entry *entry)
 		table->data_vec[entry->attr.index] = NULL;
 	/* Now this index is ready to be allocated */
 	write_unlock_irq(&table->rwlock);
-	mutex_unlock(&table->lock);
 
 	if (entry->attr.ndev)
 		dev_put(entry->attr.ndev);
 	kfree(entry);
 }
 
+static void free_gid_entry(struct kref *kref)
+{
+	struct ib_gid_table_entry *entry =
+			container_of(kref, struct ib_gid_table_entry, kref);
+
+	free_gid_entry_locked(entry);
+}
+
 /**
  * free_gid_work - Release reference to the GID entry
  * @work: Work structure to refer to GID entry which needs to be
@@ -251,7 +258,13 @@ static void free_gid_work(struct work_struct *work)
 {
 	struct ib_gid_table_entry *entry =
 		container_of(work, struct ib_gid_table_entry, del_work);
-	free_gid_entry(entry);
+	struct ib_device *device = entry->attr.device;
+	u8 port_num = entry->attr.port_num;
+	struct ib_gid_table *table = rdma_gid_table(device, port_num);
+
+	mutex_lock(&table->lock);
+	free_gid_entry_locked(entry);
+	mutex_unlock(&table->lock);
 }
 
 static struct ib_gid_table_entry *
@@ -296,6 +309,11 @@ static void put_gid_entry(struct ib_gid_table_entry *entry)
 	kref_put(&entry->kref, schedule_free_gid);
 }
 
+static void put_gid_entry_locked(struct ib_gid_table_entry *entry)
+{
+	kref_put(&entry->kref, free_gid_entry);
+}
+
 static int add_roce_gid(struct ib_gid_table_entry *entry)
 {
 	const struct ib_gid_attr *attr = &entry->attr;
@@ -398,7 +416,7 @@ static void del_gid(struct ib_device *ib_dev, u8 port,
 		table->data_vec[ix] = NULL;
 	write_unlock_irq(&table->rwlock);
 
-	put_gid_entry(entry);
+	put_gid_entry_locked(entry);
 }
 
 /* rwlock should be read locked, or lock should be held */