IB/ipath: Fix IB compliance problems with link state vs physical state

Subnet manager SetPortinfo messages distingush between changing the link
state (DOWN, ARM, ACTIVE) and the link physical state (POLL, SLEEP,
DISABLED).  These are somewhat independent commands and affect when link
width and speed changes take effect.  Without this patch, a link DOWN
physical state NOP command was causing the link width and speed settings
to take effect which should only happen when the link physical state is
goes down (either by a SMP or some link physical error like link errors
exceeding the threshold).
Signed-off-by: Ralph Campbell <ralph.campbell@qlogic.com>
Signed-off-by: Roland Dreier <rolandd@cisco.com>

drivers/infiniband/hw/ipath/ipath_common.h

@@ -75,7 +75,7 @@
 #define IPATH_IB_LINKDOWN		0
 #define IPATH_IB_LINKARM		1
 #define IPATH_IB_LINKACTIVE		2
-#define IPATH_IB_LINKINIT		3
+#define IPATH_IB_LINKDOWN_ONLY		3
 #define IPATH_IB_LINKDOWN_SLEEP		4
 #define IPATH_IB_LINKDOWN_DISABLE	5
 #define IPATH_IB_LINK_LOOPBACK	6 /* enable local loopback */

drivers/infiniband/hw/ipath/ipath_driver.c

@@ -851,8 +851,7 @@ void ipath_disarm_piobufs(struct ipath_devdata *dd, unsigned first,
  * -ETIMEDOUT state can have multiple states set, for any of several
  * transitions.
  */
-static int ipath_wait_linkstate(struct ipath_devdata *dd, u32 state,
-				int msecs)
+int ipath_wait_linkstate(struct ipath_devdata *dd, u32 state, int msecs)
 {
 	dd->ipath_state_wanted = state;
 	wait_event_interruptible_timeout(ipath_state_wait,
@@ -1656,8 +1655,8 @@ void ipath_cancel_sends(struct ipath_devdata *dd, int restore_sendctrl)
 static void ipath_set_ib_lstate(struct ipath_devdata *dd, int which)
 {
 	static const char *what[4] = {
-		[0] = "DOWN",
-		[INFINIPATH_IBCC_LINKCMD_INIT] = "INIT",
+		[0] = "NOP",
+		[INFINIPATH_IBCC_LINKCMD_DOWN] = "DOWN",
 		[INFINIPATH_IBCC_LINKCMD_ARMED] = "ARMED",
 		[INFINIPATH_IBCC_LINKCMD_ACTIVE] = "ACTIVE"
 	};
@@ -1672,9 +1671,9 @@ static void ipath_set_ib_lstate(struct ipath_devdata *dd, int which)
 			    (dd, dd->ipath_kregs->kr_ibcstatus) >>
 			    INFINIPATH_IBCS_LINKTRAININGSTATE_SHIFT) &
 			   INFINIPATH_IBCS_LINKTRAININGSTATE_MASK]);
-	/* flush all queued sends when going to DOWN or INIT, to be sure that
+	/* flush all queued sends when going to DOWN to be sure that
 	 * they don't block MAD packets */
-	if (!linkcmd || linkcmd == INFINIPATH_IBCC_LINKCMD_INIT)
+	if (linkcmd == INFINIPATH_IBCC_LINKCMD_DOWN)
 		ipath_cancel_sends(dd, 1);
 
 	ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl,
@@ -1687,6 +1686,13 @@ int ipath_set_linkstate(struct ipath_devdata *dd, u8 newstate)
 	int ret;
 
 	switch (newstate) {
+	case IPATH_IB_LINKDOWN_ONLY:
+		ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_DOWN <<
+				    INFINIPATH_IBCC_LINKCMD_SHIFT);
+		/* don't wait */
+		ret = 0;
+		goto bail;
+
 	case IPATH_IB_LINKDOWN:
 		ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKINITCMD_POLL <<
 				    INFINIPATH_IBCC_LINKINITCMD_SHIFT);
@@ -1709,16 +1715,6 @@ int ipath_set_linkstate(struct ipath_devdata *dd, u8 newstate)
 		ret = 0;
 		goto bail;
 
-	case IPATH_IB_LINKINIT:
-		if (dd->ipath_flags & IPATH_LINKINIT) {
-			ret = 0;
-			goto bail;
-		}
-		ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_INIT <<
-				    INFINIPATH_IBCC_LINKCMD_SHIFT);
-		lstate = IPATH_LINKINIT;
-		break;
-
 	case IPATH_IB_LINKARM:
 		if (dd->ipath_flags & IPATH_LINKARMED) {
 			ret = 0;

drivers/infiniband/hw/ipath/ipath_kernel.h

@@ -767,6 +767,7 @@ void ipath_kreceive(struct ipath_portdata *);
 int ipath_setrcvhdrsize(struct ipath_devdata *, unsigned);
 int ipath_reset_device(int);
 void ipath_get_faststats(unsigned long);
+int ipath_wait_linkstate(struct ipath_devdata *, u32, int);
 int ipath_set_linkstate(struct ipath_devdata *, u8);
 int ipath_set_mtu(struct ipath_devdata *, u16);
 int ipath_set_lid(struct ipath_devdata *, u32, u8);

drivers/infiniband/hw/ipath/ipath_mad.c

@@ -555,10 +555,7 @@ static int recv_subn_set_portinfo(struct ib_smp *smp,
 		/* FALLTHROUGH */
 	case IB_PORT_DOWN:
 		if (lstate == 0)
-			if (get_linkdowndefaultstate(dd))
-				lstate = IPATH_IB_LINKDOWN_SLEEP;
-			else
-				lstate = IPATH_IB_LINKDOWN;
+			lstate = IPATH_IB_LINKDOWN_ONLY;
 		else if (lstate == 1)
 			lstate = IPATH_IB_LINKDOWN_SLEEP;
 		else if (lstate == 2)
@@ -568,6 +565,8 @@ static int recv_subn_set_portinfo(struct ib_smp *smp,
 		else
 			goto err;
 		ipath_set_linkstate(dd, lstate);
+		ipath_wait_linkstate(dd, IPATH_LINKINIT | IPATH_LINKARMED |
+				IPATH_LINKACTIVE, 1000);
 		break;
 	case IB_PORT_ARMED:
 		ipath_set_linkstate(dd, IPATH_IB_LINKARM);

drivers/infiniband/hw/ipath/ipath_registers.h

@@ -185,7 +185,7 @@
 #define INFINIPATH_IBCC_LINKINITCMD_SLEEP 3
 #define INFINIPATH_IBCC_LINKINITCMD_SHIFT 16
 #define INFINIPATH_IBCC_LINKCMD_MASK 0x3ULL
-#define INFINIPATH_IBCC_LINKCMD_INIT 1	/* move to 0x11 */
+#define INFINIPATH_IBCC_LINKCMD_DOWN 1	/* move to 0x11 */
 #define INFINIPATH_IBCC_LINKCMD_ARMED 2	/* move to 0x21 */
 #define INFINIPATH_IBCC_LINKCMD_ACTIVE 3	/* move to 0x31 */
 #define INFINIPATH_IBCC_LINKCMD_SHIFT 18