[SCSI] fcoe, libfc: fully makes use of per cpu exch pool and then removes em_lock

1. Updates fcoe_rcv() to queue incoming frames to the fcoe per
   cpu thread on which this frame's exch was originated and simply
   use current cpu for request exch not originated by initiator.
   It is redundant to add this code under CONFIG_SMP, so removes
   CONFIG_SMP uses around this code.

2. Updates fc_exch_em_alloc, fc_exch_delete, fc_exch_find to use
   per cpu exch pools, here fc_exch_delete is rename of older
   fc_exch_mgr_delete_ep since ep/exch are now deleted in pools
   of EM and so brief new name is sufficient and better name.

   Updates these functions to map exch id to their index into exch
   pool using fc_cpu_mask, fc_cpu_order and EM min_xid.
   This mapping is as per detailed explanation about this in
   last patch and basically this is just as lower fc_cpu_mask
   bits of exch id as cpu number and upper bit sum of EM min_xid
   and exch index in pool.

   Uses pool next_index to keep track of exch allocation from
   pool along with pool_max_index as upper bound of exches array
   in pool.

3. Adds exch pool ptr to fc_exch to free exch to its pool in
   fc_exch_delete.

4. Updates fc_exch_mgr_reset to reset all exch pools of an EM,
   this required adding fc_exch_pool_reset func to reset exches
   in pool and then have fc_exch_mgr_reset call fc_exch_pool_reset
   for each pool within each EM for a lport.

5. Removes no longer needed exches array, em_lock, next_xid, and
   total_exches from struct fc_exch_mgr, these are not needed after
   use of per cpu exch pool, also removes not used max_read,
   last_read from struct fc_exch_mgr.

6. Updates locking notes for exch pool lock with fc_exch lock and
   uses pool lock in exch allocation, lookup and reset.
Signed-off-by: Vasu Dev <vasu.dev@intel.com>
Signed-off-by: Robert Love <robert.w.love@intel.com>
Signed-off-by: James Bottomley <James.Bottomley@suse.de>

drivers/scsi/fcoe/fcoe.c

@@ -912,8 +912,7 @@ int fcoe_rcv(struct sk_buff *skb, struct net_device *dev,
 	struct fcoe_softc *fc;
 	struct fc_frame_header *fh;
 	struct fcoe_percpu_s *fps;
-	unsigned short oxid;
-	unsigned int cpu = 0;
+	unsigned int cpu;
 
 	fc = container_of(ptype, struct fcoe_softc, fcoe_packet_type);
 	lp = fc->ctlr.lp;
@@ -947,20 +946,20 @@ int fcoe_rcv(struct sk_buff *skb, struct net_device *dev,
 	skb_set_transport_header(skb, sizeof(struct fcoe_hdr));
 	fh = (struct fc_frame_header *) skb_transport_header(skb);
 
-	oxid = ntohs(fh->fh_ox_id);
-
 	fr = fcoe_dev_from_skb(skb);
 	fr->fr_dev = lp;
 	fr->ptype = ptype;
 
-#ifdef CONFIG_SMP
 	/*
-	 * The incoming frame exchange id(oxid) is ANDed with num of online
-	 * cpu bits to get cpu and then this cpu is used for selecting
-	 * a per cpu kernel thread from fcoe_percpu.
+	 * In case the incoming frame's exchange is originated from
+	 * the initiator, then received frame's exchange id is ANDed
+	 * with fc_cpu_mask bits to get the same cpu on which exchange
+	 * was originated, otherwise just use the current cpu.
 	 */
-	cpu = oxid & (num_online_cpus() - 1);
-#endif
+	if (ntoh24(fh->fh_f_ctl) & FC_FC_EX_CTX)
+		cpu = ntohs(fh->fh_ox_id) & fc_cpu_mask;
+	else
+		cpu = smp_processor_id();
 
 	fps = &per_cpu(fcoe_percpu, cpu);
 	spin_lock_bh(&fps->fcoe_rx_list.lock);

include/scsi/libfc.h

@@ -368,6 +368,7 @@ struct fc_seq {
  */
 struct fc_exch {
 	struct fc_exch_mgr *em;		/* exchange manager */
+	struct fc_exch_pool *pool;	/* per cpu exches pool */
 	u32		state;		/* internal driver state */
 	u16		xid;		/* our exchange ID */
 	struct list_head	ex_list;	/* free or busy list linkage */
@@ -1045,10 +1046,12 @@ struct fc_exch *fc_exch_alloc(struct fc_lport *lport, struct fc_frame *fp);
  */
 struct fc_seq *fc_seq_start_next(struct fc_seq *sp);
 
+
 /*
- * Reset an exchange manager, completing all sequences and exchanges.
- * If s_id is non-zero, reset only exchanges originating from that FID.
- * If d_id is non-zero, reset only exchanges sending to that FID.
+ * Reset all EMs of a lport, releasing its all sequences and
+ * exchanges. If sid is non-zero, then reset only exchanges
+ * we sourced from that FID. If did is non-zero, reset only
+ * exchanges destined to that FID.
  */
 void fc_exch_mgr_reset(struct fc_lport *, u32 s_id, u32 d_id);