scsi: mpt3sas: Determine smp affinity on per HBA basis

Even though 'smp_affinity_enable' module parameter is enabled, if the
number of online CPUs is bigger than the number of msix vectors enabled on
that HBA, then smp affinity settings should be disabled only for this HBA.

But currently the smp affinity setting is disabled globally and hence smp
affinity will be disabled for subsequent HBAs even though number of msix
vectors enabled for this HBA matches the number of online CPU.

To fix this, define a per HBA variable smp_affinity_enable.  Initially this
variable is initialized with smp_affinity_enable module parameter value. If
this HBA has less number of msix vectors configured when compared to number
of online cpus, then only this HBA's variable smp_affinity_enable is set to
zero.
Signed-off-by: Sreekanth Reddy <sreekanth.reddy@broadcom.com>
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>

drivers/scsi/mpt3sas/mpt3sas_base.c

@@ -2813,7 +2813,7 @@ _base_free_irq(struct MPT3SAS_ADAPTER *ioc)
 
 	list_for_each_entry_safe(reply_q, next, &ioc->reply_queue_list, list) {
 		list_del(&reply_q->list);
-		if (smp_affinity_enable)
+		if (ioc->smp_affinity_enable)
 			irq_set_affinity_hint(pci_irq_vector(ioc->pdev,
 			    reply_q->msix_index), NULL);
 		free_irq(pci_irq_vector(ioc->pdev, reply_q->msix_index),
@@ -2898,7 +2898,7 @@ _base_assign_reply_queues(struct MPT3SAS_ADAPTER *ioc)
 	if (!nr_msix)
 		return;
 
-	if (smp_affinity_enable) {
+	if (ioc->smp_affinity_enable) {
 
 		/*
 		 * set irq affinity to local numa node for those irqs
@@ -3033,7 +3033,7 @@ _base_alloc_irq_vectors(struct MPT3SAS_ADAPTER *ioc)
 	struct irq_affinity desc = { .pre_vectors = ioc->high_iops_queues };
 	struct irq_affinity *descp = &desc;
 
-	if (smp_affinity_enable)
+	if (ioc->smp_affinity_enable)
 		irq_flags |= PCI_IRQ_AFFINITY;
 	else
 		descp = NULL;
@@ -3091,7 +3091,7 @@ _base_enable_msix(struct MPT3SAS_ADAPTER *ioc)
 		goto try_ioapic;
 
 	if (ioc->msix_vector_count < ioc->cpu_count)
-		smp_affinity_enable = 0;
+		ioc->smp_affinity_enable = 0;
 
 	r = _base_alloc_irq_vectors(ioc);
 	if (r < 0) {
@@ -6897,6 +6897,8 @@ mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc)
 		}
 	}
 
+	ioc->smp_affinity_enable = smp_affinity_enable;
+
 	ioc->rdpq_array_enable_assigned = 0;
 	ioc->dma_mask = 0;
 	if (ioc->is_aero_ioc)

drivers/scsi/mpt3sas/mpt3sas_base.h

@@ -1405,6 +1405,7 @@ struct MPT3SAS_ADAPTER {
 
 	u8		combined_reply_queue;
 	u8		combined_reply_index_count;
+	u8		smp_affinity_enable;
 	/* reply post register index */
 	resource_size_t	**replyPostRegisterIndex;