[BRIDGE]: forwarding table RCU

Convert the bridge forwarding database over to using RCU.
This avoids a read_lock and atomic_inc/dec in the fast path
of output.
Signed-off-by: Stephen Hemminger <shemminger@osdl.org>
Signed-off-by: David S. Miller <davem@redhat.com>

include/linux/list.h

@@ -678,6 +678,24 @@ static inline void hlist_add_after(struct hlist_node *n,
 	     pos && ({ n = pos->next; 1; }) && 				 \
 		({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
 	     pos = n)
+
+/**
+ * hlist_for_each_entry_rcu - iterate over rcu list of given type
+ * @pos:	the type * to use as a loop counter.
+ * @pos:	the &struct hlist_node to use as a loop counter.
+ * @head:	the head for your list.
+ * @member:	the name of the hlist_node within the struct.
+ *
+ * This list-traversal primitive may safely run concurrently with
+ * the _rcu list-mutation primitives such as hlist_add_rcu()
+ * as long as the traversal is guarded by rcu_read_lock().
+ */
+#define hlist_for_each_entry_rcu(tpos, pos, head, member)		 \
+	for (pos = (head)->first;					 \
+	     pos && ({ prefetch(pos->next); 1;}) &&			 \
+		({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
+	     pos = pos->next, ({ smp_read_barrier_depends(); 0; }) )
+
 #else
 #warning "don't include kernel headers in userspace"
 #endif /* __KERNEL__ */

net/bridge/br_device.c

@@ -43,10 +43,9 @@ int br_dev_xmit(struct sk_buff *skb, struct net_device *dev)
 	rcu_read_lock();
 	if (dest[0] & 1) 
 		br_flood_deliver(br, skb, 0);
-	else if ((dst = br_fdb_get(br, dest)) != NULL) {
+	else if ((dst = __br_fdb_get(br, dest)) != NULL)
 		br_deliver(dst->dst, skb);
-		br_fdb_put(dst);
-	} else
+	else
 		br_flood_deliver(br, skb, 0);
 
 	rcu_read_unlock();

net/bridge/br_fdb.c

@@ -73,7 +73,7 @@ static __inline__ int br_mac_hash(const unsigned char *mac)
 
 static __inline__ void fdb_delete(struct net_bridge_fdb_entry *f)
 {
-	hlist_del(&f->hlist);
+	hlist_del_rcu(&f->hlist);
 	if (!f->is_static)
 		list_del(&f->age_list);
 
@@ -85,7 +85,7 @@ void br_fdb_changeaddr(struct net_bridge_port *p, const unsigned char *newaddr)
 	struct net_bridge *br = p->br;
 	int i;
 	
-	write_lock_bh(&br->hash_lock);
+	spin_lock_bh(&br->hash_lock);
 
 	/* Search all chains since old address/hash is unknown */
 	for (i = 0; i < BR_HASH_SIZE; i++) {
@@ -117,7 +117,7 @@ void br_fdb_changeaddr(struct net_bridge_port *p, const unsigned char *newaddr)
 	fdb_insert(br, p, newaddr, 1);
 
 
-	write_unlock_bh(&br->hash_lock);
+	spin_unlock_bh(&br->hash_lock);
 }
 
 void br_fdb_cleanup(unsigned long _data)
@@ -126,7 +126,7 @@ void br_fdb_cleanup(unsigned long _data)
 	struct list_head *l, *n;
 	unsigned long delay;
 
-	write_lock_bh(&br->hash_lock);
+	spin_lock_bh(&br->hash_lock);
 	delay = hold_time(br);
 
 	list_for_each_safe(l, n, &br->age_list) {
@@ -144,14 +144,14 @@ void br_fdb_cleanup(unsigned long _data)
 			break;
 		}
 	}
-	write_unlock_bh(&br->hash_lock);
+	spin_unlock_bh(&br->hash_lock);
 }
 
 void br_fdb_delete_by_port(struct net_bridge *br, struct net_bridge_port *p)
 {
 	int i;
 
-	write_lock_bh(&br->hash_lock);
+	spin_lock_bh(&br->hash_lock);
 	for (i = 0; i < BR_HASH_SIZE; i++) {
 		struct hlist_node *h, *g;
 		
@@ -182,33 +182,42 @@ void br_fdb_delete_by_port(struct net_bridge *br, struct net_bridge_port *p)
 		skip_delete: ;
 		}
 	}
-	write_unlock_bh(&br->hash_lock);
+	spin_unlock_bh(&br->hash_lock);
 }
 
-struct net_bridge_fdb_entry *br_fdb_get(struct net_bridge *br, unsigned char *addr)
+/* No locking or refcounting, assumes caller has no preempt (rcu_read_lock) */
+struct net_bridge_fdb_entry *__br_fdb_get(struct net_bridge *br,
+					  const unsigned char *addr)
 {
 	struct hlist_node *h;
+	struct net_bridge_fdb_entry *fdb;
 
-	read_lock_bh(&br->hash_lock);
-		
-	hlist_for_each(h, &br->hash[br_mac_hash(addr)]) {
-		struct net_bridge_fdb_entry *fdb
-			= hlist_entry(h, struct net_bridge_fdb_entry, hlist);
-
+	hlist_for_each_entry_rcu(fdb, h, &br->hash[br_mac_hash(addr)], hlist) {
 		if (!memcmp(fdb->addr.addr, addr, ETH_ALEN)) {
-			if (has_expired(br, fdb))
-				goto ret_null;
-
-			atomic_inc(&fdb->use_count);
-			read_unlock_bh(&br->hash_lock);
+			if (unlikely(has_expired(br, fdb)))
+				break;
 			return fdb;
 		}
 	}
- ret_null:
-	read_unlock_bh(&br->hash_lock);
+
 	return NULL;
 }
 
+/* Interface used by ATM hook that keeps a ref count */
+struct net_bridge_fdb_entry *br_fdb_get(struct net_bridge *br, 
+					unsigned char *addr)
+{
+	struct net_bridge_fdb_entry *fdb;
+
+	rcu_read_lock();
+	fdb = __br_fdb_get(br, addr);
+	if (fdb) 
+		atomic_inc(&fdb->use_count);
+	rcu_read_unlock();
+	return fdb;
+}
+
+
 void br_fdb_put(struct net_bridge_fdb_entry *ent)
 {
 	if (atomic_dec_and_test(&ent->use_count))
@@ -229,9 +238,9 @@ int br_fdb_fillbuf(struct net_bridge *br, void *buf,
 
 	memset(buf, 0, maxnum*sizeof(struct __fdb_entry));
 
-	read_lock_bh(&br->hash_lock);
+	rcu_read_lock();
 	for (i = 0; i < BR_HASH_SIZE; i++) {
-		hlist_for_each_entry(f, h, &br->hash[i], hlist) {
+		hlist_for_each_entry_rcu(f, h, &br->hash[i], hlist) {
 			if (num >= maxnum)
 				goto out;
 
@@ -255,7 +264,7 @@ int br_fdb_fillbuf(struct net_bridge *br, void *buf,
 	}
 
  out:
-	read_unlock_bh(&br->hash_lock);
+	rcu_read_unlock();
 
 	return num;
 }
@@ -309,7 +318,7 @@ static int fdb_insert(struct net_bridge *br, struct net_bridge_port *source,
 
 	memcpy(fdb->addr.addr, addr, ETH_ALEN);
 	atomic_set(&fdb->use_count, 1);
-	hlist_add_head(&fdb->hlist, &br->hash[hash]);
+	hlist_add_head_rcu(&fdb->hlist, &br->hash[hash]);
 
 	if (!timer_pending(&br->gc_timer)) {
 		br->gc_timer.expires = jiffies + hold_time(br);
@@ -332,8 +341,8 @@ int br_fdb_insert(struct net_bridge *br, struct net_bridge_port *source,
 {
 	int ret;
 
-	write_lock_bh(&br->hash_lock);
+	spin_lock_bh(&br->hash_lock);
 	ret = fdb_insert(br, source, addr, is_local);
-	write_unlock_bh(&br->hash_lock);
+	spin_unlock_bh(&br->hash_lock);
 	return ret;
 }

net/bridge/br_if.c

@@ -149,7 +149,7 @@ static struct net_device *new_bridge_dev(const char *name)
 
 	br->lock = SPIN_LOCK_UNLOCKED;
 	INIT_LIST_HEAD(&br->port_list);
-	br->hash_lock = RW_LOCK_UNLOCKED;
+	br->hash_lock = SPIN_LOCK_UNLOCKED;
 
 	br->bridge_id.prio[0] = 0x80;
 	br->bridge_id.prio[1] = 0x00;

net/bridge/br_input.c

@@ -83,19 +83,17 @@ int br_handle_frame_finish(struct sk_buff *skb)
 		goto out;
 	}
 
-	dst = br_fdb_get(br, dest);
+	dst = __br_fdb_get(br, dest);
 	if (dst != NULL && dst->is_local) {
 		if (!passedup)
 			br_pass_frame_up(br, skb);
 		else
 			kfree_skb(skb);
-		br_fdb_put(dst);
 		goto out;
 	}
 
 	if (dst != NULL) {
 		br_forward(dst->dst, skb);
-		br_fdb_put(dst);
 		goto out;
 	}
 

net/bridge/br_private.h

@@ -86,7 +86,7 @@ struct net_bridge
 	struct list_head		port_list;
 	struct net_device		*dev;
 	struct net_device_stats		statistics;
-	rwlock_t			hash_lock;
+	spinlock_t			hash_lock;
 	struct hlist_head		hash[BR_HASH_SIZE];
 	struct list_head		age_list;
 
@@ -136,8 +136,10 @@ extern void br_fdb_changeaddr(struct net_bridge_port *p,
 extern void br_fdb_cleanup(unsigned long arg);
 extern void br_fdb_delete_by_port(struct net_bridge *br,
 			   struct net_bridge_port *p);
+extern struct net_bridge_fdb_entry *__br_fdb_get(struct net_bridge *br,
+						 const unsigned char *addr);
 extern struct net_bridge_fdb_entry *br_fdb_get(struct net_bridge *br,
-					unsigned char *addr);
+					       unsigned char *addr);
 extern void br_fdb_put(struct net_bridge_fdb_entry *ent);
 extern int br_fdb_fillbuf(struct net_bridge *br, void *buf, 
 			  unsigned long count, unsigned long off);