Commit 1faf2894 authored by Srinivas Eeda's avatar Srinivas Eeda Committed by Mark Fasheh

ocfs2_dlm: disallow a domain join if node maps mismatch

There is a small window where a joining node may not see the node(s) that
just died but are still part of the domain. To fix this, we must disallow
join requests if the joining node has a different node map.

A new field node_map is added to dlm_query_join_request to send the current
nodes nodemap along with join request. On the receiving end the nodes that
are part of the cluster verifies if this new node sees all the nodes that
are still part of the cluster. They disallow the join if the maps mismatch.
Signed-off-by: default avatarSrinivas Eeda <srinivas.eeda@oracle.com>
Signed-off-by: default avatarMark Fasheh <mark.fasheh@oracle.com>
parent f3f85464
...@@ -38,6 +38,9 @@ ...@@ -38,6 +38,9 @@
* locking semantics of the file system using the protocol. It should * locking semantics of the file system using the protocol. It should
* be somewhere else, I'm sure, but right now it isn't. * be somewhere else, I'm sure, but right now it isn't.
* *
* New in version 7:
* - DLM join domain includes the live nodemap
*
* New in version 6: * New in version 6:
* - DLM lockres remote refcount fixes. * - DLM lockres remote refcount fixes.
* *
...@@ -54,7 +57,7 @@ ...@@ -54,7 +57,7 @@
* - full 64 bit i_size in the metadata lock lvbs * - full 64 bit i_size in the metadata lock lvbs
* - introduction of "rw" lock and pushing meta/data locking down * - introduction of "rw" lock and pushing meta/data locking down
*/ */
#define O2NET_PROTOCOL_VERSION 6ULL #define O2NET_PROTOCOL_VERSION 7ULL
struct o2net_handshake { struct o2net_handshake {
__be64 protocol_version; __be64 protocol_version;
__be64 connector_id; __be64 connector_id;
......
...@@ -625,12 +625,16 @@ struct dlm_begin_reco ...@@ -625,12 +625,16 @@ struct dlm_begin_reco
}; };
#define BITS_PER_BYTE 8
#define BITS_TO_BYTES(bits) (((bits)+BITS_PER_BYTE-1)/BITS_PER_BYTE)
struct dlm_query_join_request struct dlm_query_join_request
{ {
u8 node_idx; u8 node_idx;
u8 pad1[2]; u8 pad1[2];
u8 name_len; u8 name_len;
u8 domain[O2NM_MAX_NAME_LEN]; u8 domain[O2NM_MAX_NAME_LEN];
u8 node_map[BITS_TO_BYTES(O2NM_MAX_NODES)];
}; };
struct dlm_assert_joined struct dlm_assert_joined
......
...@@ -48,6 +48,36 @@ ...@@ -48,6 +48,36 @@
#define MLOG_MASK_PREFIX (ML_DLM|ML_DLM_DOMAIN) #define MLOG_MASK_PREFIX (ML_DLM|ML_DLM_DOMAIN)
#include "cluster/masklog.h" #include "cluster/masklog.h"
/*
* ocfs2 node maps are array of long int, which limits to send them freely
* across the wire due to endianness issues. To workaround this, we convert
* long ints to byte arrays. Following 3 routines are helper functions to
* set/test/copy bits within those array of bytes
*/
static inline void byte_set_bit(u8 nr, u8 map[])
{
map[nr >> 3] |= (1UL << (nr & 7));
}
static inline int byte_test_bit(u8 nr, u8 map[])
{
return ((1UL << (nr & 7)) & (map[nr >> 3])) != 0;
}
static inline void byte_copymap(u8 dmap[], unsigned long smap[],
unsigned int sz)
{
unsigned int nn;
if (!sz)
return;
memset(dmap, 0, ((sz + 7) >> 3));
for (nn = 0 ; nn < sz; nn++)
if (test_bit(nn, smap))
byte_set_bit(nn, dmap);
}
static void dlm_free_pagevec(void **vec, int pages) static void dlm_free_pagevec(void **vec, int pages)
{ {
while (pages--) while (pages--)
...@@ -641,6 +671,7 @@ static int dlm_query_join_handler(struct o2net_msg *msg, u32 len, void *data, ...@@ -641,6 +671,7 @@ static int dlm_query_join_handler(struct o2net_msg *msg, u32 len, void *data,
struct dlm_query_join_request *query; struct dlm_query_join_request *query;
enum dlm_query_join_response response; enum dlm_query_join_response response;
struct dlm_ctxt *dlm = NULL; struct dlm_ctxt *dlm = NULL;
u8 nodenum;
query = (struct dlm_query_join_request *) msg->buf; query = (struct dlm_query_join_request *) msg->buf;
...@@ -664,6 +695,25 @@ static int dlm_query_join_handler(struct o2net_msg *msg, u32 len, void *data, ...@@ -664,6 +695,25 @@ static int dlm_query_join_handler(struct o2net_msg *msg, u32 len, void *data,
spin_lock(&dlm_domain_lock); spin_lock(&dlm_domain_lock);
dlm = __dlm_lookup_domain_full(query->domain, query->name_len); dlm = __dlm_lookup_domain_full(query->domain, query->name_len);
if (!dlm)
goto unlock_respond;
/*
* There is a small window where the joining node may not see the
* node(s) that just left but still part of the cluster. DISALLOW
* join request if joining node has different node map.
*/
nodenum=0;
while (nodenum < O2NM_MAX_NODES) {
if (test_bit(nodenum, dlm->domain_map)) {
if (!byte_test_bit(nodenum, query->node_map)) {
response = JOIN_DISALLOW;
goto unlock_respond;
}
}
nodenum++;
}
/* Once the dlm ctxt is marked as leaving then we don't want /* Once the dlm ctxt is marked as leaving then we don't want
* to be put in someone's domain map. * to be put in someone's domain map.
* Also, explicitly disallow joining at certain troublesome * Also, explicitly disallow joining at certain troublesome
...@@ -705,6 +755,7 @@ static int dlm_query_join_handler(struct o2net_msg *msg, u32 len, void *data, ...@@ -705,6 +755,7 @@ static int dlm_query_join_handler(struct o2net_msg *msg, u32 len, void *data,
spin_unlock(&dlm->spinlock); spin_unlock(&dlm->spinlock);
} }
unlock_respond:
spin_unlock(&dlm_domain_lock); spin_unlock(&dlm_domain_lock);
respond: respond:
...@@ -854,6 +905,9 @@ static int dlm_request_join(struct dlm_ctxt *dlm, ...@@ -854,6 +905,9 @@ static int dlm_request_join(struct dlm_ctxt *dlm,
join_msg.name_len = strlen(dlm->name); join_msg.name_len = strlen(dlm->name);
memcpy(join_msg.domain, dlm->name, join_msg.name_len); memcpy(join_msg.domain, dlm->name, join_msg.name_len);
/* copy live node map to join message */
byte_copymap(join_msg.node_map, dlm->live_nodes_map, O2NM_MAX_NODES);
status = o2net_send_message(DLM_QUERY_JOIN_MSG, DLM_MOD_KEY, &join_msg, status = o2net_send_message(DLM_QUERY_JOIN_MSG, DLM_MOD_KEY, &join_msg,
sizeof(join_msg), node, &retval); sizeof(join_msg), node, &retval);
if (status < 0 && status != -ENOPROTOOPT) { if (status < 0 && status != -ENOPROTOOPT) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment