crypto: qat - add support for broadcasting mode

Add support for broadcasting mode in firmware loader to enable the next
generation of QAT devices.
Signed-off-by: Jack Xu <jack.xu@intel.com>
Co-developed-by: Wojciech Ziemba <wojciech.ziemba@intel.com>
Signed-off-by: Wojciech Ziemba <wojciech.ziemba@intel.com>
Reviewed-by: Giovanni Cabiddu <giovanni.cabiddu@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h

@@ -15,6 +15,7 @@ struct icp_qat_fw_loader_ae_data {
 struct icp_qat_fw_loader_hal_handle {
 	struct icp_qat_fw_loader_ae_data aes[ICP_QAT_UCLO_MAX_AE];
 	unsigned int ae_mask;
+	unsigned int admin_ae_mask;
 	unsigned int slice_mask;
 	unsigned int revision_id;
 	unsigned int ae_max_num;

drivers/crypto/qat/qat_common/icp_qat_hal.h

@@ -53,6 +53,15 @@ enum fcu_csr {
 	FCU_RAMBASE_ADDR_LO   = 0x8d8
 };
 
+enum fcu_csr_4xxx {
+	FCU_CONTROL_4XXX           = 0x1000,
+	FCU_STATUS_4XXX            = 0x1004,
+	FCU_ME_BROADCAST_MASK_TYPE = 0x1008,
+	FCU_AE_LOADED_4XXX         = 0x1010,
+	FCU_DRAM_ADDR_LO_4XXX      = 0x1014,
+	FCU_DRAM_ADDR_HI_4XXX      = 0x1018,
+};
+
 enum fcu_cmd {
 	FCU_CTRL_CMD_NOOP  = 0,
 	FCU_CTRL_CMD_AUTH  = 1,
@@ -90,6 +99,7 @@ enum fcu_sts {
 #define LCS_STATUS          (0x1)
 #define MMC_SHARE_CS_BITPOS         2
 #define WAKEUP_EVENT 0x10000
+#define FCU_CTRL_BROADCAST_POS   0x4
 #define FCU_CTRL_AE_POS     0x8
 #define FCU_AUTH_STS_MASK   0x7
 #define FCU_STS_DONE_POS    0x9

drivers/crypto/qat/qat_common/qat_hal.c

@@ -762,6 +762,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle,
 	handle->pci_dev = pci_info->pci_dev;
 	handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid;
 	handle->hal_handle->ae_mask = hw_data->ae_mask;
+	handle->hal_handle->admin_ae_mask = hw_data->admin_ae_mask;
 	handle->hal_handle->slice_mask = hw_data->accel_mask;
 	handle->cfg_ae_mask = ALL_AE_MASK;
 	/* create AE objects */

drivers/crypto/qat/qat_common/qat_uclo.c

@@ -1239,6 +1239,83 @@ static int qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle,
 	return -EINVAL;
 }
 
+static bool qat_uclo_is_broadcast(struct icp_qat_fw_loader_handle *handle,
+				  int imgid)
+{
+	struct icp_qat_suof_handle *sobj_handle;
+
+	if (!handle->chip_info->tgroup_share_ustore)
+		return false;
+
+	sobj_handle = (struct icp_qat_suof_handle *)handle->sobj_handle;
+	if (handle->hal_handle->admin_ae_mask &
+	    sobj_handle->img_table.simg_hdr[imgid].ae_mask)
+		return false;
+
+	return true;
+}
+
+static int qat_uclo_broadcast_load_fw(struct icp_qat_fw_loader_handle *handle,
+				      struct icp_qat_fw_auth_desc *desc)
+{
+	unsigned long ae_mask = handle->hal_handle->ae_mask;
+	unsigned long desc_ae_mask = desc->ae_mask;
+	u32 fcu_sts, ae_broadcast_mask = 0;
+	u32 fcu_loaded_csr, ae_loaded;
+	u32 fcu_sts_csr, fcu_ctl_csr;
+	unsigned int ae, retry = 0;
+
+	if (handle->chip_info->tgroup_share_ustore) {
+		fcu_ctl_csr = handle->chip_info->fcu_ctl_csr;
+		fcu_sts_csr = handle->chip_info->fcu_sts_csr;
+		fcu_loaded_csr = handle->chip_info->fcu_loaded_ae_csr;
+	} else {
+		pr_err("Chip 0x%x doesn't support broadcast load\n",
+		       handle->pci_dev->device);
+		return -EINVAL;
+	}
+
+	for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
+		if (qat_hal_check_ae_active(handle, (unsigned char)ae)) {
+			pr_err("QAT: Broadcast load failed. AE is not enabled or active.\n");
+			return -EINVAL;
+		}
+
+		if (test_bit(ae, &desc_ae_mask))
+			ae_broadcast_mask |= 1 << ae;
+	}
+
+	if (ae_broadcast_mask) {
+		SET_CAP_CSR(handle, FCU_ME_BROADCAST_MASK_TYPE,
+			    ae_broadcast_mask);
+
+		SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_LOAD);
+
+		do {
+			msleep(FW_AUTH_WAIT_PERIOD);
+			fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr);
+			fcu_sts &= FCU_AUTH_STS_MASK;
+
+			if (fcu_sts == FCU_STS_LOAD_FAIL) {
+				pr_err("Broadcast load failed: 0x%x)\n", fcu_sts);
+				return -EINVAL;
+			} else if (fcu_sts == FCU_STS_LOAD_DONE) {
+				ae_loaded = GET_CAP_CSR(handle, fcu_loaded_csr);
+				ae_loaded >>= handle->chip_info->fcu_loaded_ae_pos;
+
+				if ((ae_loaded & ae_broadcast_mask) == ae_broadcast_mask)
+					break;
+			}
+		} while (retry++ < FW_AUTH_MAX_RETRY);
+
+		if (retry > FW_AUTH_MAX_RETRY) {
+			pr_err("QAT: broadcast load failed timeout %d\n", retry);
+			return -EINVAL;
+		}
+	}
+	return 0;
+}
+
 static int qat_uclo_simg_alloc(struct icp_qat_fw_loader_handle *handle,
 			       struct icp_firml_dram_desc *dram_desc,
 			       unsigned int size)
@@ -1420,7 +1497,9 @@ static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle,
 			return -EINVAL;
 		}
 		SET_CAP_CSR(handle, fcu_ctl_csr,
-			    (FCU_CTRL_CMD_LOAD | (i << FCU_CTRL_AE_POS)));
+			    (FCU_CTRL_CMD_LOAD |
+			    (1 << FCU_CTRL_BROADCAST_POS) |
+			    (i << FCU_CTRL_AE_POS)));
 
 		do {
 			msleep(FW_AUTH_WAIT_PERIOD);
@@ -1945,8 +2024,13 @@ static int qat_uclo_wr_suof_img(struct icp_qat_fw_loader_handle *handle)
 			goto wr_err;
 		if (qat_uclo_auth_fw(handle, desc))
 			goto wr_err;
-		if (qat_uclo_load_fw(handle, desc))
-			goto wr_err;
+		if (qat_uclo_is_broadcast(handle, i)) {
+			if (qat_uclo_broadcast_load_fw(handle, desc))
+				goto wr_err;
+		} else {
+			if (qat_uclo_load_fw(handle, desc))
+				goto wr_err;
+		}
 		qat_uclo_ummap_auth_fw(handle, &desc);
 	}
 	return 0;