Commit c1ecb90a authored by Chris Leech's avatar Chris Leech Committed by James Bottomley

[SCSI] libfc: reduce hold time on SCSI host lock

Introduce a new lock to protect the list of fc_fcp_pkt structs in libfc
instead of using the host lock.  This reduces the contention of this heavily
used lock, and I see up to a 25% performance gain in CPU bound small I/O
tests when scaling out across multiple quad-core CPUs.

The big win is in removing the host lock from the completion path
completely, as it does not need to be held around the call to scsi_done.
Signed-off-by: default avatarChris Leech <christopher.leech@intel.com>
Signed-off-by: default avatarRobert Love <robert.w.love@intel.com>
Signed-off-by: default avatarJames Bottomley <James.Bottomley@suse.de>
parent 5543c72e
...@@ -68,18 +68,20 @@ struct kmem_cache *scsi_pkt_cachep; ...@@ -68,18 +68,20 @@ struct kmem_cache *scsi_pkt_cachep;
/** /**
* struct fc_fcp_internal - FCP layer internal data * struct fc_fcp_internal - FCP layer internal data
* @scsi_pkt_pool: Memory pool to draw FCP packets from * @scsi_pkt_pool: Memory pool to draw FCP packets from
* @scsi_queue_lock: Protects the scsi_pkt_queue
* @scsi_pkt_queue: Current FCP packets * @scsi_pkt_queue: Current FCP packets
* @last_can_queue_ramp_down_time: ramp down time * @last_can_queue_ramp_down_time: ramp down time
* @last_can_queue_ramp_up_time: ramp up time * @last_can_queue_ramp_up_time: ramp up time
* @max_can_queue: max can_queue size * @max_can_queue: max can_queue size
*/ */
struct fc_fcp_internal { struct fc_fcp_internal {
mempool_t *scsi_pkt_pool; mempool_t *scsi_pkt_pool;
struct list_head scsi_pkt_queue; spinlock_t scsi_queue_lock;
unsigned long last_can_queue_ramp_down_time; struct list_head scsi_pkt_queue;
unsigned long last_can_queue_ramp_up_time; unsigned long last_can_queue_ramp_down_time;
int max_can_queue; unsigned long last_can_queue_ramp_up_time;
int max_can_queue;
}; };
#define fc_get_scsi_internal(x) ((struct fc_fcp_internal *)(x)->scsi_priv) #define fc_get_scsi_internal(x) ((struct fc_fcp_internal *)(x)->scsi_priv)
...@@ -410,12 +412,14 @@ static inline struct fc_frame *fc_fcp_frame_alloc(struct fc_lport *lport, ...@@ -410,12 +412,14 @@ static inline struct fc_frame *fc_fcp_frame_alloc(struct fc_lport *lport,
unsigned long flags; unsigned long flags;
fp = fc_frame_alloc(lport, len); fp = fc_frame_alloc(lport, len);
if (!fp) { if (likely(fp))
spin_lock_irqsave(lport->host->host_lock, flags); return fp;
fc_fcp_can_queue_ramp_down(lport);
spin_unlock_irqrestore(lport->host->host_lock, flags); /* error case */
} spin_lock_irqsave(lport->host->host_lock, flags);
return fp; fc_fcp_can_queue_ramp_down(lport);
spin_unlock_irqrestore(lport->host->host_lock, flags);
return NULL;
} }
/** /**
...@@ -990,7 +994,7 @@ static void fc_fcp_cleanup_each_cmd(struct fc_lport *lport, unsigned int id, ...@@ -990,7 +994,7 @@ static void fc_fcp_cleanup_each_cmd(struct fc_lport *lport, unsigned int id,
struct scsi_cmnd *sc_cmd; struct scsi_cmnd *sc_cmd;
unsigned long flags; unsigned long flags;
spin_lock_irqsave(lport->host->host_lock, flags); spin_lock_irqsave(&si->scsi_queue_lock, flags);
restart: restart:
list_for_each_entry(fsp, &si->scsi_pkt_queue, list) { list_for_each_entry(fsp, &si->scsi_pkt_queue, list) {
sc_cmd = fsp->cmd; sc_cmd = fsp->cmd;
...@@ -1001,7 +1005,7 @@ static void fc_fcp_cleanup_each_cmd(struct fc_lport *lport, unsigned int id, ...@@ -1001,7 +1005,7 @@ static void fc_fcp_cleanup_each_cmd(struct fc_lport *lport, unsigned int id,
continue; continue;
fc_fcp_pkt_hold(fsp); fc_fcp_pkt_hold(fsp);
spin_unlock_irqrestore(lport->host->host_lock, flags); spin_unlock_irqrestore(&si->scsi_queue_lock, flags);
if (!fc_fcp_lock_pkt(fsp)) { if (!fc_fcp_lock_pkt(fsp)) {
fc_fcp_cleanup_cmd(fsp, error); fc_fcp_cleanup_cmd(fsp, error);
...@@ -1010,14 +1014,14 @@ static void fc_fcp_cleanup_each_cmd(struct fc_lport *lport, unsigned int id, ...@@ -1010,14 +1014,14 @@ static void fc_fcp_cleanup_each_cmd(struct fc_lport *lport, unsigned int id,
} }
fc_fcp_pkt_release(fsp); fc_fcp_pkt_release(fsp);
spin_lock_irqsave(lport->host->host_lock, flags); spin_lock_irqsave(&si->scsi_queue_lock, flags);
/* /*
* while we dropped the lock multiple pkts could * while we dropped the lock multiple pkts could
* have been released, so we have to start over. * have been released, so we have to start over.
*/ */
goto restart; goto restart;
} }
spin_unlock_irqrestore(lport->host->host_lock, flags); spin_unlock_irqrestore(&si->scsi_queue_lock, flags);
} }
/** /**
...@@ -1035,11 +1039,12 @@ static void fc_fcp_abort_io(struct fc_lport *lport) ...@@ -1035,11 +1039,12 @@ static void fc_fcp_abort_io(struct fc_lport *lport)
* @fsp: The FCP packet to send * @fsp: The FCP packet to send
* *
* Return: Zero for success and -1 for failure * Return: Zero for success and -1 for failure
* Locks: Called with the host lock and irqs disabled. * Locks: Called without locks held
*/ */
static int fc_fcp_pkt_send(struct fc_lport *lport, struct fc_fcp_pkt *fsp) static int fc_fcp_pkt_send(struct fc_lport *lport, struct fc_fcp_pkt *fsp)
{ {
struct fc_fcp_internal *si = fc_get_scsi_internal(lport); struct fc_fcp_internal *si = fc_get_scsi_internal(lport);
unsigned long flags;
int rc; int rc;
fsp->cmd->SCp.ptr = (char *)fsp; fsp->cmd->SCp.ptr = (char *)fsp;
...@@ -1049,13 +1054,16 @@ static int fc_fcp_pkt_send(struct fc_lport *lport, struct fc_fcp_pkt *fsp) ...@@ -1049,13 +1054,16 @@ static int fc_fcp_pkt_send(struct fc_lport *lport, struct fc_fcp_pkt *fsp)
int_to_scsilun(fsp->cmd->device->lun, int_to_scsilun(fsp->cmd->device->lun,
(struct scsi_lun *)fsp->cdb_cmd.fc_lun); (struct scsi_lun *)fsp->cdb_cmd.fc_lun);
memcpy(fsp->cdb_cmd.fc_cdb, fsp->cmd->cmnd, fsp->cmd->cmd_len); memcpy(fsp->cdb_cmd.fc_cdb, fsp->cmd->cmnd, fsp->cmd->cmd_len);
list_add_tail(&fsp->list, &si->scsi_pkt_queue);
spin_unlock_irq(lport->host->host_lock); spin_lock_irqsave(&si->scsi_queue_lock, flags);
list_add_tail(&fsp->list, &si->scsi_pkt_queue);
spin_unlock_irqrestore(&si->scsi_queue_lock, flags);
rc = lport->tt.fcp_cmd_send(lport, fsp, fc_fcp_recv); rc = lport->tt.fcp_cmd_send(lport, fsp, fc_fcp_recv);
spin_lock_irq(lport->host->host_lock); if (unlikely(rc)) {
if (rc) spin_lock_irqsave(&si->scsi_queue_lock, flags);
list_del(&fsp->list); list_del(&fsp->list);
spin_unlock_irqrestore(&si->scsi_queue_lock, flags);
}
return rc; return rc;
} }
...@@ -1752,6 +1760,7 @@ int fc_queuecommand(struct scsi_cmnd *sc_cmd, void (*done)(struct scsi_cmnd *)) ...@@ -1752,6 +1760,7 @@ int fc_queuecommand(struct scsi_cmnd *sc_cmd, void (*done)(struct scsi_cmnd *))
struct fcoe_dev_stats *stats; struct fcoe_dev_stats *stats;
lport = shost_priv(sc_cmd->device->host); lport = shost_priv(sc_cmd->device->host);
spin_unlock_irq(lport->host->host_lock);
rval = fc_remote_port_chkready(rport); rval = fc_remote_port_chkready(rport);
if (rval) { if (rval) {
...@@ -1834,6 +1843,7 @@ int fc_queuecommand(struct scsi_cmnd *sc_cmd, void (*done)(struct scsi_cmnd *)) ...@@ -1834,6 +1843,7 @@ int fc_queuecommand(struct scsi_cmnd *sc_cmd, void (*done)(struct scsi_cmnd *))
rc = SCSI_MLQUEUE_HOST_BUSY; rc = SCSI_MLQUEUE_HOST_BUSY;
} }
out: out:
spin_lock_irq(lport->host->host_lock);
return rc; return rc;
} }
EXPORT_SYMBOL(fc_queuecommand); EXPORT_SYMBOL(fc_queuecommand);
...@@ -1864,11 +1874,8 @@ static void fc_io_compl(struct fc_fcp_pkt *fsp) ...@@ -1864,11 +1874,8 @@ static void fc_io_compl(struct fc_fcp_pkt *fsp)
lport = fsp->lp; lport = fsp->lp;
si = fc_get_scsi_internal(lport); si = fc_get_scsi_internal(lport);
spin_lock_irqsave(lport->host->host_lock, flags); if (!fsp->cmd)
if (!fsp->cmd) {
spin_unlock_irqrestore(lport->host->host_lock, flags);
return; return;
}
/* /*
* if can_queue ramp down is done then try can_queue ramp up * if can_queue ramp down is done then try can_queue ramp up
...@@ -1880,10 +1887,8 @@ static void fc_io_compl(struct fc_fcp_pkt *fsp) ...@@ -1880,10 +1887,8 @@ static void fc_io_compl(struct fc_fcp_pkt *fsp)
sc_cmd = fsp->cmd; sc_cmd = fsp->cmd;
fsp->cmd = NULL; fsp->cmd = NULL;
if (!sc_cmd->SCp.ptr) { if (!sc_cmd->SCp.ptr)
spin_unlock_irqrestore(lport->host->host_lock, flags);
return; return;
}
CMD_SCSI_STATUS(sc_cmd) = fsp->cdb_status; CMD_SCSI_STATUS(sc_cmd) = fsp->cdb_status;
switch (fsp->status_code) { switch (fsp->status_code) {
...@@ -1945,10 +1950,11 @@ static void fc_io_compl(struct fc_fcp_pkt *fsp) ...@@ -1945,10 +1950,11 @@ static void fc_io_compl(struct fc_fcp_pkt *fsp)
break; break;
} }
spin_lock_irqsave(&si->scsi_queue_lock, flags);
list_del(&fsp->list); list_del(&fsp->list);
spin_unlock_irqrestore(&si->scsi_queue_lock, flags);
sc_cmd->SCp.ptr = NULL; sc_cmd->SCp.ptr = NULL;
sc_cmd->scsi_done(sc_cmd); sc_cmd->scsi_done(sc_cmd);
spin_unlock_irqrestore(lport->host->host_lock, flags);
/* release ref from initial allocation in queue command */ /* release ref from initial allocation in queue command */
fc_fcp_pkt_release(fsp); fc_fcp_pkt_release(fsp);
...@@ -2216,6 +2222,7 @@ int fc_fcp_init(struct fc_lport *lport) ...@@ -2216,6 +2222,7 @@ int fc_fcp_init(struct fc_lport *lport)
lport->scsi_priv = si; lport->scsi_priv = si;
si->max_can_queue = lport->host->can_queue; si->max_can_queue = lport->host->can_queue;
INIT_LIST_HEAD(&si->scsi_pkt_queue); INIT_LIST_HEAD(&si->scsi_pkt_queue);
spin_lock_init(&si->scsi_queue_lock);
si->scsi_pkt_pool = mempool_create_slab_pool(2, scsi_pkt_cachep); si->scsi_pkt_pool = mempool_create_slab_pool(2, scsi_pkt_cachep);
if (!si->scsi_pkt_pool) { if (!si->scsi_pkt_pool) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment