drbd_state.c 53.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
/*
   drbd_state.c

   This file is part of DRBD by Philipp Reisner and Lars Ellenberg.

   Copyright (C) 2001-2008, LINBIT Information Technologies GmbH.
   Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>.
   Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.

   Thanks to Carter Burden, Bart Grantham and Gennadiy Nerubayev
   from Logicworks, Inc. for making SDP replication support possible.

   drbd is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2, or (at your option)
   any later version.

   drbd is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with drbd; see the file COPYING.  If not, write to
   the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
 */

#include <linux/drbd_limits.h>
#include "drbd_int.h"
#include "drbd_req.h"

32
/* in drbd_main.c */
33
extern void tl_abort_disk_io(struct drbd_conf *mdev);
34

35 36 37 38 39 40 41 42
struct after_state_chg_work {
	struct drbd_work w;
	union drbd_state os;
	union drbd_state ns;
	enum chg_state_flags flags;
	struct completion *done;
};

43 44 45 46 47 48 49 50 51
enum sanitize_state_warnings {
	NO_WARNING,
	ABORTED_ONLINE_VERIFY,
	ABORTED_RESYNC,
	CONNECTION_LOST_NEGOTIATING,
	IMPLICITLY_UPGRADED_DISK,
	IMPLICITLY_UPGRADED_PDSK,
};

52
static int w_after_state_ch(struct drbd_work *w, int unused);
53 54
static void after_state_ch(struct drbd_conf *mdev, union drbd_state os,
			   union drbd_state ns, enum chg_state_flags flags);
55
static enum drbd_state_rv is_valid_state(struct drbd_conf *, union drbd_state);
56
static enum drbd_state_rv is_valid_soft_transition(union drbd_state, union drbd_state, struct drbd_tconn *);
57
static enum drbd_state_rv is_valid_transition(union drbd_state os, union drbd_state ns);
58
static union drbd_state sanitize_state(struct drbd_conf *mdev, union drbd_state ns,
59
				       enum sanitize_state_warnings *warn);
60

61 62 63 64 65
static inline bool is_susp(union drbd_state s)
{
        return s.susp || s.susp_nod || s.susp_fen;
}

66
bool conn_all_vols_unconf(struct drbd_tconn *tconn)
67 68
{
	struct drbd_conf *mdev;
69
	bool rv = true;
70
	int vnr;
71

72
	rcu_read_lock();
73
	idr_for_each_entry(&tconn->volumes, mdev, vnr) {
74 75
		if (mdev->state.disk != D_DISKLESS ||
		    mdev->state.conn != C_STANDALONE ||
76 77 78 79
		    mdev->state.role != R_SECONDARY) {
			rv = false;
			break;
		}
80
	}
81 82 83
	rcu_read_unlock();

	return rv;
84 85
}

86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110
/* Unfortunately the states where not correctly ordered, when
   they where defined. therefore can not use max_t() here. */
static enum drbd_role max_role(enum drbd_role role1, enum drbd_role role2)
{
	if (role1 == R_PRIMARY || role2 == R_PRIMARY)
		return R_PRIMARY;
	if (role1 == R_SECONDARY || role2 == R_SECONDARY)
		return R_SECONDARY;
	return R_UNKNOWN;
}
static enum drbd_role min_role(enum drbd_role role1, enum drbd_role role2)
{
	if (role1 == R_UNKNOWN || role2 == R_UNKNOWN)
		return R_UNKNOWN;
	if (role1 == R_SECONDARY || role2 == R_SECONDARY)
		return R_SECONDARY;
	return R_PRIMARY;
}

enum drbd_role conn_highest_role(struct drbd_tconn *tconn)
{
	enum drbd_role role = R_UNKNOWN;
	struct drbd_conf *mdev;
	int vnr;

111
	rcu_read_lock();
112 113
	idr_for_each_entry(&tconn->volumes, mdev, vnr)
		role = max_role(role, mdev->state.role);
114
	rcu_read_unlock();
115 116 117 118 119 120 121 122 123 124

	return role;
}

enum drbd_role conn_highest_peer(struct drbd_tconn *tconn)
{
	enum drbd_role peer = R_UNKNOWN;
	struct drbd_conf *mdev;
	int vnr;

125
	rcu_read_lock();
126 127
	idr_for_each_entry(&tconn->volumes, mdev, vnr)
		peer = max_role(peer, mdev->state.peer);
128
	rcu_read_unlock();
129 130 131 132 133 134 135 136 137 138

	return peer;
}

enum drbd_disk_state conn_highest_disk(struct drbd_tconn *tconn)
{
	enum drbd_disk_state ds = D_DISKLESS;
	struct drbd_conf *mdev;
	int vnr;

139
	rcu_read_lock();
140 141
	idr_for_each_entry(&tconn->volumes, mdev, vnr)
		ds = max_t(enum drbd_disk_state, ds, mdev->state.disk);
142
	rcu_read_unlock();
143 144 145 146

	return ds;
}

147 148 149 150 151 152
enum drbd_disk_state conn_lowest_disk(struct drbd_tconn *tconn)
{
	enum drbd_disk_state ds = D_MASK;
	struct drbd_conf *mdev;
	int vnr;

153
	rcu_read_lock();
154 155
	idr_for_each_entry(&tconn->volumes, mdev, vnr)
		ds = min_t(enum drbd_disk_state, ds, mdev->state.disk);
156
	rcu_read_unlock();
157 158 159 160

	return ds;
}

161 162 163 164 165 166
enum drbd_disk_state conn_highest_pdsk(struct drbd_tconn *tconn)
{
	enum drbd_disk_state ds = D_DISKLESS;
	struct drbd_conf *mdev;
	int vnr;

167
	rcu_read_lock();
168 169
	idr_for_each_entry(&tconn->volumes, mdev, vnr)
		ds = max_t(enum drbd_disk_state, ds, mdev->state.pdsk);
170
	rcu_read_unlock();
171 172 173 174

	return ds;
}

175 176 177 178 179 180
enum drbd_conns conn_lowest_conn(struct drbd_tconn *tconn)
{
	enum drbd_conns conn = C_MASK;
	struct drbd_conf *mdev;
	int vnr;

181
	rcu_read_lock();
182 183
	idr_for_each_entry(&tconn->volumes, mdev, vnr)
		conn = min_t(enum drbd_conns, conn, mdev->state.conn);
184
	rcu_read_unlock();
185 186 187 188

	return conn;
}

189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206
static bool no_peer_wf_report_params(struct drbd_tconn *tconn)
{
	struct drbd_conf *mdev;
	int vnr;
	bool rv = true;

	rcu_read_lock();
	idr_for_each_entry(&tconn->volumes, mdev, vnr)
		if (mdev->state.conn == C_WF_REPORT_PARAMS) {
			rv = false;
			break;
		}
	rcu_read_unlock();

	return rv;
}


207 208 209 210 211 212 213 214 215 216 217 218 219
/**
 * cl_wide_st_chg() - true if the state change is a cluster wide one
 * @mdev:	DRBD device.
 * @os:		old (current) state.
 * @ns:		new (wanted) state.
 */
static int cl_wide_st_chg(struct drbd_conf *mdev,
			  union drbd_state os, union drbd_state ns)
{
	return (os.conn >= C_CONNECTED && ns.conn >= C_CONNECTED &&
		 ((os.role != R_PRIMARY && ns.role == R_PRIMARY) ||
		  (os.conn != C_STARTING_SYNC_T && ns.conn == C_STARTING_SYNC_T) ||
		  (os.conn != C_STARTING_SYNC_S && ns.conn == C_STARTING_SYNC_S) ||
Lars Ellenberg's avatar
Lars Ellenberg committed
220
		  (os.disk != D_FAILED && ns.disk == D_FAILED))) ||
221
		(os.conn >= C_CONNECTED && ns.conn == C_DISCONNECTING) ||
222 223
		(os.conn == C_CONNECTED && ns.conn == C_VERIFY_S) ||
		(os.conn == C_CONNECTED && ns.conn == C_WF_REPORT_PARAMS);
224 225
}

226 227 228 229 230 231 232 233
static union drbd_state
apply_mask_val(union drbd_state os, union drbd_state mask, union drbd_state val)
{
	union drbd_state ns;
	ns.i = (os.i & ~mask.i) | val.i;
	return ns;
}

234 235 236 237 238
enum drbd_state_rv
drbd_change_state(struct drbd_conf *mdev, enum chg_state_flags f,
		  union drbd_state mask, union drbd_state val)
{
	unsigned long flags;
239
	union drbd_state ns;
240 241 242
	enum drbd_state_rv rv;

	spin_lock_irqsave(&mdev->tconn->req_lock, flags);
243
	ns = apply_mask_val(drbd_read_state(mdev), mask, val);
244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276
	rv = _drbd_set_state(mdev, ns, f, NULL);
	spin_unlock_irqrestore(&mdev->tconn->req_lock, flags);

	return rv;
}

/**
 * drbd_force_state() - Impose a change which happens outside our control on our state
 * @mdev:	DRBD device.
 * @mask:	mask of state bits to change.
 * @val:	value of new state bits.
 */
void drbd_force_state(struct drbd_conf *mdev,
	union drbd_state mask, union drbd_state val)
{
	drbd_change_state(mdev, CS_HARD, mask, val);
}

static enum drbd_state_rv
_req_st_cond(struct drbd_conf *mdev, union drbd_state mask,
	     union drbd_state val)
{
	union drbd_state os, ns;
	unsigned long flags;
	enum drbd_state_rv rv;

	if (test_and_clear_bit(CL_ST_CHG_SUCCESS, &mdev->flags))
		return SS_CW_SUCCESS;

	if (test_and_clear_bit(CL_ST_CHG_FAIL, &mdev->flags))
		return SS_CW_FAILED_BY_PEER;

	spin_lock_irqsave(&mdev->tconn->req_lock, flags);
277
	os = drbd_read_state(mdev);
278
	ns = sanitize_state(mdev, apply_mask_val(os, mask, val), NULL);
279 280 281
	rv = is_valid_transition(os, ns);
	if (rv == SS_SUCCESS)
		rv = SS_UNKNOWN_ERROR;  /* cont waiting, otherwise fail. */
282 283 284

	if (!cl_wide_st_chg(mdev, os, ns))
		rv = SS_CW_NO_NEED;
285
	if (rv == SS_UNKNOWN_ERROR) {
286 287
		rv = is_valid_state(mdev, ns);
		if (rv == SS_SUCCESS) {
288
			rv = is_valid_soft_transition(os, ns, mdev->tconn);
289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319
			if (rv == SS_SUCCESS)
				rv = SS_UNKNOWN_ERROR; /* cont waiting, otherwise fail. */
		}
	}
	spin_unlock_irqrestore(&mdev->tconn->req_lock, flags);

	return rv;
}

/**
 * drbd_req_state() - Perform an eventually cluster wide state change
 * @mdev:	DRBD device.
 * @mask:	mask of state bits to change.
 * @val:	value of new state bits.
 * @f:		flags
 *
 * Should not be called directly, use drbd_request_state() or
 * _drbd_request_state().
 */
static enum drbd_state_rv
drbd_req_state(struct drbd_conf *mdev, union drbd_state mask,
	       union drbd_state val, enum chg_state_flags f)
{
	struct completion done;
	unsigned long flags;
	union drbd_state os, ns;
	enum drbd_state_rv rv;

	init_completion(&done);

	if (f & CS_SERIALIZE)
320
		mutex_lock(mdev->state_mutex);
321 322

	spin_lock_irqsave(&mdev->tconn->req_lock, flags);
323
	os = drbd_read_state(mdev);
324
	ns = sanitize_state(mdev, apply_mask_val(os, mask, val), NULL);
325
	rv = is_valid_transition(os, ns);
326 327
	if (rv < SS_SUCCESS) {
		spin_unlock_irqrestore(&mdev->tconn->req_lock, flags);
328
		goto abort;
329
	}
330 331 332 333

	if (cl_wide_st_chg(mdev, os, ns)) {
		rv = is_valid_state(mdev, ns);
		if (rv == SS_SUCCESS)
334
			rv = is_valid_soft_transition(os, ns, mdev->tconn);
335 336 337 338 339 340 341 342
		spin_unlock_irqrestore(&mdev->tconn->req_lock, flags);

		if (rv < SS_SUCCESS) {
			if (f & CS_VERBOSE)
				print_st_err(mdev, os, ns, rv);
			goto abort;
		}

343
		if (drbd_send_state_req(mdev, mask, val)) {
344 345 346 347 348 349 350 351 352 353 354 355 356 357 358
			rv = SS_CW_FAILED_BY_PEER;
			if (f & CS_VERBOSE)
				print_st_err(mdev, os, ns, rv);
			goto abort;
		}

		wait_event(mdev->state_wait,
			(rv = _req_st_cond(mdev, mask, val)));

		if (rv < SS_SUCCESS) {
			if (f & CS_VERBOSE)
				print_st_err(mdev, os, ns, rv);
			goto abort;
		}
		spin_lock_irqsave(&mdev->tconn->req_lock, flags);
359
		ns = apply_mask_val(drbd_read_state(mdev), mask, val);
360 361 362 363 364 365 366 367 368 369 370 371 372 373
		rv = _drbd_set_state(mdev, ns, f, &done);
	} else {
		rv = _drbd_set_state(mdev, ns, f, &done);
	}

	spin_unlock_irqrestore(&mdev->tconn->req_lock, flags);

	if (f & CS_WAIT_COMPLETE && rv == SS_SUCCESS) {
		D_ASSERT(current != mdev->tconn->worker.task);
		wait_for_completion(&done);
	}

abort:
	if (f & CS_SERIALIZE)
374
		mutex_unlock(mdev->state_mutex);
375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428

	return rv;
}

/**
 * _drbd_request_state() - Request a state change (with flags)
 * @mdev:	DRBD device.
 * @mask:	mask of state bits to change.
 * @val:	value of new state bits.
 * @f:		flags
 *
 * Cousin of drbd_request_state(), useful with the CS_WAIT_COMPLETE
 * flag, or when logging of failed state change requests is not desired.
 */
enum drbd_state_rv
_drbd_request_state(struct drbd_conf *mdev, union drbd_state mask,
		    union drbd_state val, enum chg_state_flags f)
{
	enum drbd_state_rv rv;

	wait_event(mdev->state_wait,
		   (rv = drbd_req_state(mdev, mask, val, f)) != SS_IN_TRANSIENT_STATE);

	return rv;
}

static void print_st(struct drbd_conf *mdev, char *name, union drbd_state ns)
{
	dev_err(DEV, " %s = { cs:%s ro:%s/%s ds:%s/%s %c%c%c%c%c%c }\n",
	    name,
	    drbd_conn_str(ns.conn),
	    drbd_role_str(ns.role),
	    drbd_role_str(ns.peer),
	    drbd_disk_str(ns.disk),
	    drbd_disk_str(ns.pdsk),
	    is_susp(ns) ? 's' : 'r',
	    ns.aftr_isp ? 'a' : '-',
	    ns.peer_isp ? 'p' : '-',
	    ns.user_isp ? 'u' : '-',
	    ns.susp_fen ? 'F' : '-',
	    ns.susp_nod ? 'N' : '-'
	    );
}

void print_st_err(struct drbd_conf *mdev, union drbd_state os,
	          union drbd_state ns, enum drbd_state_rv err)
{
	if (err == SS_IN_TRANSIENT_STATE)
		return;
	dev_err(DEV, "State change failed: %s\n", drbd_set_st_err_str(err));
	print_st(mdev, " state", os);
	print_st(mdev, "wanted", ns);
}

429
static long print_state_change(char *pb, union drbd_state os, union drbd_state ns,
430 431
			       enum chg_state_flags flags)
{
432
	char *pbp;
433 434
	pbp = pb;
	*pbp = 0;
435

436
	if (ns.role != os.role && flags & CS_DC_ROLE)
437 438 439
		pbp += sprintf(pbp, "role( %s -> %s ) ",
			       drbd_role_str(os.role),
			       drbd_role_str(ns.role));
440
	if (ns.peer != os.peer && flags & CS_DC_PEER)
441 442 443
		pbp += sprintf(pbp, "peer( %s -> %s ) ",
			       drbd_role_str(os.peer),
			       drbd_role_str(ns.peer));
444
	if (ns.conn != os.conn && flags & CS_DC_CONN)
445 446 447
		pbp += sprintf(pbp, "conn( %s -> %s ) ",
			       drbd_conn_str(os.conn),
			       drbd_conn_str(ns.conn));
448
	if (ns.disk != os.disk && flags & CS_DC_DISK)
449 450 451
		pbp += sprintf(pbp, "disk( %s -> %s ) ",
			       drbd_disk_str(os.disk),
			       drbd_disk_str(ns.disk));
452
	if (ns.pdsk != os.pdsk && flags & CS_DC_PDSK)
453 454 455
		pbp += sprintf(pbp, "pdsk( %s -> %s ) ",
			       drbd_disk_str(os.pdsk),
			       drbd_disk_str(ns.pdsk));
456 457 458 459 460 461 462 463 464 465 466 467

	return pbp - pb;
}

static void drbd_pr_state_change(struct drbd_conf *mdev, union drbd_state os, union drbd_state ns,
				 enum chg_state_flags flags)
{
	char pb[300];
	char *pbp = pb;

	pbp += print_state_change(pbp, os, ns, flags ^ CS_DC_MASK);

468 469 470 471 472 473 474 475 476 477 478 479
	if (ns.aftr_isp != os.aftr_isp)
		pbp += sprintf(pbp, "aftr_isp( %d -> %d ) ",
			       os.aftr_isp,
			       ns.aftr_isp);
	if (ns.peer_isp != os.peer_isp)
		pbp += sprintf(pbp, "peer_isp( %d -> %d ) ",
			       os.peer_isp,
			       ns.peer_isp);
	if (ns.user_isp != os.user_isp)
		pbp += sprintf(pbp, "user_isp( %d -> %d ) ",
			       os.user_isp,
			       ns.user_isp);
480

481
	if (pbp != pb)
482 483
		dev_info(DEV, "%s\n", pb);
}
484

485 486 487 488
static void conn_pr_state_change(struct drbd_tconn *tconn, union drbd_state os, union drbd_state ns,
				 enum chg_state_flags flags)
{
	char pb[300];
489 490 491
	char *pbp = pb;

	pbp += print_state_change(pbp, os, ns, flags);
492

493 494 495 496 497 498
	if (is_susp(ns) != is_susp(os) && flags & CS_DC_SUSP)
		pbp += sprintf(pbp, "susp( %d -> %d ) ",
			       is_susp(os),
			       is_susp(ns));

	if (pbp != pb)
499 500 501 502
		conn_info(tconn, "%s\n", pb);
}


503 504 505 506 507 508 509 510 511 512 513 514
/**
 * is_valid_state() - Returns an SS_ error code if ns is not valid
 * @mdev:	DRBD device.
 * @ns:		State to consider.
 */
static enum drbd_state_rv
is_valid_state(struct drbd_conf *mdev, union drbd_state ns)
{
	/* See drbd_state_sw_errors in drbd_strings.c */

	enum drbd_fencing_p fp;
	enum drbd_state_rv rv = SS_SUCCESS;
515
	struct net_conf *nc;
516

Philipp Reisner's avatar
Philipp Reisner committed
517
	rcu_read_lock();
518 519
	fp = FP_DONT_CARE;
	if (get_ldev(mdev)) {
Philipp Reisner's avatar
Philipp Reisner committed
520
		fp = rcu_dereference(mdev->ldev->disk_conf)->fencing;
521 522 523
		put_ldev(mdev);
	}

524 525 526
	nc = rcu_dereference(mdev->tconn->net_conf);
	if (nc) {
		if (!nc->two_primaries && ns.role == R_PRIMARY) {
527 528
			if (ns.peer == R_PRIMARY)
				rv = SS_TWO_PRIMARIES;
529
			else if (conn_highest_peer(mdev->tconn) == R_PRIMARY)
530
				rv = SS_O_VOL_PEER_PRI;
531
		}
532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565
	}

	if (rv <= 0)
		/* already found a reason to abort */;
	else if (ns.role == R_SECONDARY && mdev->open_cnt)
		rv = SS_DEVICE_IN_USE;

	else if (ns.role == R_PRIMARY && ns.conn < C_CONNECTED && ns.disk < D_UP_TO_DATE)
		rv = SS_NO_UP_TO_DATE_DISK;

	else if (fp >= FP_RESOURCE &&
		 ns.role == R_PRIMARY && ns.conn < C_CONNECTED && ns.pdsk >= D_UNKNOWN)
		rv = SS_PRIMARY_NOP;

	else if (ns.role == R_PRIMARY && ns.disk <= D_INCONSISTENT && ns.pdsk <= D_INCONSISTENT)
		rv = SS_NO_UP_TO_DATE_DISK;

	else if (ns.conn > C_CONNECTED && ns.disk < D_INCONSISTENT)
		rv = SS_NO_LOCAL_DISK;

	else if (ns.conn > C_CONNECTED && ns.pdsk < D_INCONSISTENT)
		rv = SS_NO_REMOTE_DISK;

	else if (ns.conn > C_CONNECTED && ns.disk < D_UP_TO_DATE && ns.pdsk < D_UP_TO_DATE)
		rv = SS_NO_UP_TO_DATE_DISK;

	else if ((ns.conn == C_CONNECTED ||
		  ns.conn == C_WF_BITMAP_S ||
		  ns.conn == C_SYNC_SOURCE ||
		  ns.conn == C_PAUSED_SYNC_S) &&
		  ns.disk == D_OUTDATED)
		rv = SS_CONNECTED_OUTDATES;

	else if ((ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T) &&
566
		 (nc->verify_alg[0] == 0))
567 568 569 570 571 572 573 574 575
		rv = SS_NO_VERIFY_ALG;

	else if ((ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T) &&
		  mdev->tconn->agreed_pro_version < 88)
		rv = SS_NOT_SUPPORTED;

	else if (ns.conn >= C_CONNECTED && ns.pdsk == D_UNKNOWN)
		rv = SS_CONNECTED_OUTDATES;

576 577
	rcu_read_unlock();

578 579 580 581
	return rv;
}

/**
582
 * is_valid_soft_transition() - Returns an SS_ error code if the state transition is not possible
583 584
 * This function limits state transitions that may be declined by DRBD. I.e.
 * user requests (aka soft transitions).
585 586 587 588 589
 * @mdev:	DRBD device.
 * @ns:		new state.
 * @os:		old state.
 */
static enum drbd_state_rv
590
is_valid_soft_transition(union drbd_state os, union drbd_state ns, struct drbd_tconn *tconn)
591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612
{
	enum drbd_state_rv rv = SS_SUCCESS;

	if ((ns.conn == C_STARTING_SYNC_T || ns.conn == C_STARTING_SYNC_S) &&
	    os.conn > C_CONNECTED)
		rv = SS_RESYNC_RUNNING;

	if (ns.conn == C_DISCONNECTING && os.conn == C_STANDALONE)
		rv = SS_ALREADY_STANDALONE;

	if (ns.disk > D_ATTACHING && os.disk == D_DISKLESS)
		rv = SS_IS_DISKLESS;

	if (ns.conn == C_WF_CONNECTION && os.conn < C_UNCONNECTED)
		rv = SS_NO_NET_CONFIG;

	if (ns.disk == D_OUTDATED && os.disk < D_OUTDATED && os.disk != D_ATTACHING)
		rv = SS_LOWER_THAN_OUTDATED;

	if (ns.conn == C_DISCONNECTING && os.conn == C_UNCONNECTED)
		rv = SS_IN_TRANSIENT_STATE;

613 614
	/* if (ns.conn == os.conn && ns.conn == C_WF_REPORT_PARAMS)
	   rv = SS_IN_TRANSIENT_STATE; */
615

616 617 618 619 620 621 622
	/* While establishing a connection only allow cstate to change.
	   Delay/refuse role changes, detach attach etc... */
	if (test_bit(STATE_SENT, &tconn->flags) &&
	    !(os.conn == C_WF_REPORT_PARAMS ||
	      (ns.conn == C_WF_REPORT_PARAMS && os.conn == C_WF_CONNECTION)))
		rv = SS_IN_TRANSIENT_STATE;

623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640
	if ((ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T) && os.conn < C_CONNECTED)
		rv = SS_NEED_CONNECTION;

	if ((ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T) &&
	    ns.conn != os.conn && os.conn > C_CONNECTED)
		rv = SS_RESYNC_RUNNING;

	if ((ns.conn == C_STARTING_SYNC_S || ns.conn == C_STARTING_SYNC_T) &&
	    os.conn < C_CONNECTED)
		rv = SS_NEED_CONNECTION;

	if ((ns.conn == C_SYNC_TARGET || ns.conn == C_SYNC_SOURCE)
	    && os.conn < C_WF_REPORT_PARAMS)
		rv = SS_NEED_CONNECTION; /* No NetworkFailure -> SyncTarget etc... */

	return rv;
}

641
static enum drbd_state_rv
642
is_valid_conn_transition(enum drbd_conns oc, enum drbd_conns nc)
643
{
644 645 646
	/* no change -> nothing to do, at least for the connection part */
	if (oc == nc)
		return SS_NOTHING_TO_DO;
647

648 649 650 651 652 653 654
	/* disconnect of an unconfigured connection does not make sense */
	if (oc == C_STANDALONE && nc == C_DISCONNECTING)
		return SS_ALREADY_STANDALONE;

	/* from C_STANDALONE, we start with C_UNCONNECTED */
	if (oc == C_STANDALONE && nc != C_UNCONNECTED)
		return SS_NEED_CONNECTION;
655

656 657 658 659 660
	/* When establishing a connection we need to go through WF_REPORT_PARAMS!
	   Necessary to do the right thing upon invalidate-remote on a disconnected resource */
	if (oc < C_WF_REPORT_PARAMS && nc >= C_CONNECTED)
		return SS_NEED_CONNECTION;

661
	/* After a network error only C_UNCONNECTED or C_DISCONNECTING may follow. */
662
	if (oc >= C_TIMEOUT && oc <= C_TEAR_DOWN && nc != C_UNCONNECTED && nc != C_DISCONNECTING)
663
		return SS_IN_TRANSIENT_STATE;
664 665

	/* After C_DISCONNECTING only C_STANDALONE may follow */
666
	if (oc == C_DISCONNECTING && nc != C_STANDALONE)
667
		return SS_IN_TRANSIENT_STATE;
668

669
	return SS_SUCCESS;
670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687
}


/**
 * is_valid_transition() - Returns an SS_ error code if the state transition is not possible
 * This limits hard state transitions. Hard state transitions are facts there are
 * imposed on DRBD by the environment. E.g. disk broke or network broke down.
 * But those hard state transitions are still not allowed to do everything.
 * @ns:		new state.
 * @os:		old state.
 */
static enum drbd_state_rv
is_valid_transition(union drbd_state os, union drbd_state ns)
{
	enum drbd_state_rv rv;

	rv = is_valid_conn_transition(os.conn, ns.conn);

688 689 690 691 692 693 694
	/* we cannot fail (again) if we already detached */
	if (ns.disk == D_FAILED && os.disk == D_DISKLESS)
		rv = SS_IS_DISKLESS;

	return rv;
}

695 696 697 698 699 700 701 702 703 704 705 706 707 708 709
static void print_sanitize_warnings(struct drbd_conf *mdev, enum sanitize_state_warnings warn)
{
	static const char *msg_table[] = {
		[NO_WARNING] = "",
		[ABORTED_ONLINE_VERIFY] = "Online-verify aborted.",
		[ABORTED_RESYNC] = "Resync aborted.",
		[CONNECTION_LOST_NEGOTIATING] = "Connection lost while negotiating, no data!",
		[IMPLICITLY_UPGRADED_DISK] = "Implicitly upgraded disk",
		[IMPLICITLY_UPGRADED_PDSK] = "Implicitly upgraded pdsk",
	};

	if (warn != NO_WARNING)
		dev_warn(DEV, "%s\n", msg_table[warn]);
}

710 711 712 713 714 715 716 717 718 719
/**
 * sanitize_state() - Resolves implicitly necessary additional changes to a state transition
 * @mdev:	DRBD device.
 * @os:		old state.
 * @ns:		new state.
 * @warn_sync_abort:
 *
 * When we loose connection, we have to set the state of the peers disk (pdsk)
 * to D_UNKNOWN. This rule and many more along those lines are in this function.
 */
720
static union drbd_state sanitize_state(struct drbd_conf *mdev, union drbd_state ns,
721
				       enum sanitize_state_warnings *warn)
722 723 724 725
{
	enum drbd_fencing_p fp;
	enum drbd_disk_state disk_min, disk_max, pdsk_min, pdsk_max;

726 727 728
	if (warn)
		*warn = NO_WARNING;

729 730
	fp = FP_DONT_CARE;
	if (get_ldev(mdev)) {
Philipp Reisner's avatar
Philipp Reisner committed
731 732 733
		rcu_read_lock();
		fp = rcu_dereference(mdev->ldev->disk_conf)->fencing;
		rcu_read_unlock();
734 735 736
		put_ldev(mdev);
	}

737
	/* Implications from connection to peer and peer_isp */
738 739 740 741 742 743 744 745 746 747 748
	if (ns.conn < C_CONNECTED) {
		ns.peer_isp = 0;
		ns.peer = R_UNKNOWN;
		if (ns.pdsk > D_UNKNOWN || ns.pdsk < D_INCONSISTENT)
			ns.pdsk = D_UNKNOWN;
	}

	/* Clear the aftr_isp when becoming unconfigured */
	if (ns.conn == C_STANDALONE && ns.disk == D_DISKLESS && ns.role == R_SECONDARY)
		ns.aftr_isp = 0;

749
	/* An implication of the disk states onto the connection state */
750
	/* Abort resync if a disk fails/detaches */
751
	if (ns.conn > C_CONNECTED && (ns.disk <= D_FAILED || ns.pdsk <= D_FAILED)) {
752 753 754
		if (warn)
			*warn = ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T ?
				ABORTED_ONLINE_VERIFY : ABORTED_RESYNC;
755 756 757 758 759 760 761 762 763 764
		ns.conn = C_CONNECTED;
	}

	/* Connection breaks down before we finished "Negotiating" */
	if (ns.conn < C_CONNECTED && ns.disk == D_NEGOTIATING &&
	    get_ldev_if_state(mdev, D_NEGOTIATING)) {
		if (mdev->ed_uuid == mdev->ldev->md.uuid[UI_CURRENT]) {
			ns.disk = mdev->new_state_tmp.disk;
			ns.pdsk = mdev->new_state_tmp.pdsk;
		} else {
765 766
			if (warn)
				*warn = CONNECTION_LOST_NEGOTIATING;
767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847
			ns.disk = D_DISKLESS;
			ns.pdsk = D_UNKNOWN;
		}
		put_ldev(mdev);
	}

	/* D_CONSISTENT and D_OUTDATED vanish when we get connected */
	if (ns.conn >= C_CONNECTED && ns.conn < C_AHEAD) {
		if (ns.disk == D_CONSISTENT || ns.disk == D_OUTDATED)
			ns.disk = D_UP_TO_DATE;
		if (ns.pdsk == D_CONSISTENT || ns.pdsk == D_OUTDATED)
			ns.pdsk = D_UP_TO_DATE;
	}

	/* Implications of the connection stat on the disk states */
	disk_min = D_DISKLESS;
	disk_max = D_UP_TO_DATE;
	pdsk_min = D_INCONSISTENT;
	pdsk_max = D_UNKNOWN;
	switch ((enum drbd_conns)ns.conn) {
	case C_WF_BITMAP_T:
	case C_PAUSED_SYNC_T:
	case C_STARTING_SYNC_T:
	case C_WF_SYNC_UUID:
	case C_BEHIND:
		disk_min = D_INCONSISTENT;
		disk_max = D_OUTDATED;
		pdsk_min = D_UP_TO_DATE;
		pdsk_max = D_UP_TO_DATE;
		break;
	case C_VERIFY_S:
	case C_VERIFY_T:
		disk_min = D_UP_TO_DATE;
		disk_max = D_UP_TO_DATE;
		pdsk_min = D_UP_TO_DATE;
		pdsk_max = D_UP_TO_DATE;
		break;
	case C_CONNECTED:
		disk_min = D_DISKLESS;
		disk_max = D_UP_TO_DATE;
		pdsk_min = D_DISKLESS;
		pdsk_max = D_UP_TO_DATE;
		break;
	case C_WF_BITMAP_S:
	case C_PAUSED_SYNC_S:
	case C_STARTING_SYNC_S:
	case C_AHEAD:
		disk_min = D_UP_TO_DATE;
		disk_max = D_UP_TO_DATE;
		pdsk_min = D_INCONSISTENT;
		pdsk_max = D_CONSISTENT; /* D_OUTDATED would be nice. But explicit outdate necessary*/
		break;
	case C_SYNC_TARGET:
		disk_min = D_INCONSISTENT;
		disk_max = D_INCONSISTENT;
		pdsk_min = D_UP_TO_DATE;
		pdsk_max = D_UP_TO_DATE;
		break;
	case C_SYNC_SOURCE:
		disk_min = D_UP_TO_DATE;
		disk_max = D_UP_TO_DATE;
		pdsk_min = D_INCONSISTENT;
		pdsk_max = D_INCONSISTENT;
		break;
	case C_STANDALONE:
	case C_DISCONNECTING:
	case C_UNCONNECTED:
	case C_TIMEOUT:
	case C_BROKEN_PIPE:
	case C_NETWORK_FAILURE:
	case C_PROTOCOL_ERROR:
	case C_TEAR_DOWN:
	case C_WF_CONNECTION:
	case C_WF_REPORT_PARAMS:
	case C_MASK:
		break;
	}
	if (ns.disk > disk_max)
		ns.disk = disk_max;

	if (ns.disk < disk_min) {
848 849
		if (warn)
			*warn = IMPLICITLY_UPGRADED_DISK;
850 851 852 853 854 855
		ns.disk = disk_min;
	}
	if (ns.pdsk > pdsk_max)
		ns.pdsk = pdsk_max;

	if (ns.pdsk < pdsk_min) {
856 857
		if (warn)
			*warn = IMPLICITLY_UPGRADED_PDSK;
858 859 860 861
		ns.pdsk = pdsk_min;
	}

	if (fp == FP_STONITH &&
862
	    (ns.role == R_PRIMARY && ns.conn < C_CONNECTED && ns.pdsk > D_OUTDATED))
863 864
		ns.susp_fen = 1; /* Suspend IO while fence-peer handler runs (peer lost) */

865
	if (mdev->tconn->res_opts.on_no_data == OND_SUSPEND_IO &&
866
	    (ns.role == R_PRIMARY && ns.disk < D_UP_TO_DATE && ns.pdsk < D_UP_TO_DATE))
867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931
		ns.susp_nod = 1; /* Suspend IO while no data available (no accessible data available) */

	if (ns.aftr_isp || ns.peer_isp || ns.user_isp) {
		if (ns.conn == C_SYNC_SOURCE)
			ns.conn = C_PAUSED_SYNC_S;
		if (ns.conn == C_SYNC_TARGET)
			ns.conn = C_PAUSED_SYNC_T;
	} else {
		if (ns.conn == C_PAUSED_SYNC_S)
			ns.conn = C_SYNC_SOURCE;
		if (ns.conn == C_PAUSED_SYNC_T)
			ns.conn = C_SYNC_TARGET;
	}

	return ns;
}

void drbd_resume_al(struct drbd_conf *mdev)
{
	if (test_and_clear_bit(AL_SUSPENDED, &mdev->flags))
		dev_info(DEV, "Resumed AL updates\n");
}

/* helper for __drbd_set_state */
static void set_ov_position(struct drbd_conf *mdev, enum drbd_conns cs)
{
	if (mdev->tconn->agreed_pro_version < 90)
		mdev->ov_start_sector = 0;
	mdev->rs_total = drbd_bm_bits(mdev);
	mdev->ov_position = 0;
	if (cs == C_VERIFY_T) {
		/* starting online verify from an arbitrary position
		 * does not fit well into the existing protocol.
		 * on C_VERIFY_T, we initialize ov_left and friends
		 * implicitly in receive_DataRequest once the
		 * first P_OV_REQUEST is received */
		mdev->ov_start_sector = ~(sector_t)0;
	} else {
		unsigned long bit = BM_SECT_TO_BIT(mdev->ov_start_sector);
		if (bit >= mdev->rs_total) {
			mdev->ov_start_sector =
				BM_BIT_TO_SECT(mdev->rs_total - 1);
			mdev->rs_total = 1;
		} else
			mdev->rs_total -= bit;
		mdev->ov_position = mdev->ov_start_sector;
	}
	mdev->ov_left = mdev->rs_total;
}

/**
 * __drbd_set_state() - Set a new DRBD state
 * @mdev:	DRBD device.
 * @ns:		new state.
 * @flags:	Flags
 * @done:	Optional completion, that will get completed after the after_state_ch() finished
 *
 * Caller needs to hold req_lock, and global_state_lock. Do not call directly.
 */
enum drbd_state_rv
__drbd_set_state(struct drbd_conf *mdev, union drbd_state ns,
	         enum chg_state_flags flags, struct completion *done)
{
	union drbd_state os;
	enum drbd_state_rv rv = SS_SUCCESS;
932
	enum sanitize_state_warnings ssw;
933 934
	struct after_state_chg_work *ascw;

935
	os = drbd_read_state(mdev);
936

937
	ns = sanitize_state(mdev, ns, &ssw);
938 939 940
	if (ns.i == os.i)
		return SS_NOTHING_TO_DO;

941 942 943 944
	rv = is_valid_transition(os, ns);
	if (rv < SS_SUCCESS)
		return rv;

945 946 947 948 949 950 951 952 953 954
	if (!(flags & CS_HARD)) {
		/*  pre-state-change checks ; only look at ns  */
		/* See drbd_state_sw_errors in drbd_strings.c */

		rv = is_valid_state(mdev, ns);
		if (rv < SS_SUCCESS) {
			/* If the old state was illegal as well, then let
			   this happen...*/

			if (is_valid_state(mdev, os) == rv)
955
				rv = is_valid_soft_transition(os, ns, mdev->tconn);
956
		} else
957
			rv = is_valid_soft_transition(os, ns, mdev->tconn);
958 959 960 961 962 963 964 965
	}

	if (rv < SS_SUCCESS) {
		if (flags & CS_VERBOSE)
			print_st_err(mdev, os, ns, rv);
		return rv;
	}

966
	print_sanitize_warnings(mdev, ssw);
967

968
	drbd_pr_state_change(mdev, os, ns, flags);
969

970 971 972 973 974 975
	/* Display changes to the susp* flags that where caused by the call to
	   sanitize_state(). Only display it here if we where not called from
	   _conn_request_state() */
	if (!(flags & CS_DC_SUSP))
		conn_pr_state_change(mdev->tconn, os, ns, (flags & ~CS_DC_MASK) | CS_DC_SUSP);

976 977 978 979 980 981 982 983
	/* if we are going -> D_FAILED or D_DISKLESS, grab one extra reference
	 * on the ldev here, to be sure the transition -> D_DISKLESS resp.
	 * drbd_ldev_destroy() won't happen before our corresponding
	 * after_state_ch works run, where we put_ldev again. */
	if ((os.disk != D_FAILED && ns.disk == D_FAILED) ||
	    (os.disk != D_DISKLESS && ns.disk == D_DISKLESS))
		atomic_inc(&mdev->local_cnt);

984
	mdev->state.i = ns.i;
985 986 987
	mdev->tconn->susp = ns.susp;
	mdev->tconn->susp_nod = ns.susp_nod;
	mdev->tconn->susp_fen = ns.susp_fen;
988 989 990 991

	if (os.disk == D_ATTACHING && ns.disk >= D_NEGOTIATING)
		drbd_print_uuids(mdev, "attached to UUIDs");

992 993 994 995 996
	/* Wake up role changes, that were delayed because of connection establishing */
	if (os.conn == C_WF_REPORT_PARAMS && ns.conn != C_WF_REPORT_PARAMS &&
	    no_peer_wf_report_params(mdev->tconn))
		clear_bit(STATE_SENT, &mdev->tconn->flags);

997 998
	wake_up(&mdev->misc_wait);
	wake_up(&mdev->state_wait);
999
	wake_up(&mdev->tconn->ping_wait);
1000

1001 1002
	/* Aborted verify run, or we reached the stop sector.
	 * Log the last position, unless end-of-device. */
1003
	if ((os.conn == C_VERIFY_S || os.conn == C_VERIFY_T) &&
1004
	    ns.conn <= C_CONNECTED) {
1005 1006
		mdev->ov_start_sector =
			BM_BIT_TO_SECT(drbd_bm_bits(mdev) - mdev->ov_left);
1007 1008 1009
		if (mdev->ov_left)
			dev_info(DEV, "Online Verify reached sector %llu\n",
				(unsigned long long)mdev->ov_start_sector);
1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057
	}

	if ((os.conn == C_PAUSED_SYNC_T || os.conn == C_PAUSED_SYNC_S) &&
	    (ns.conn == C_SYNC_TARGET  || ns.conn == C_SYNC_SOURCE)) {
		dev_info(DEV, "Syncer continues.\n");
		mdev->rs_paused += (long)jiffies
				  -(long)mdev->rs_mark_time[mdev->rs_last_mark];
		if (ns.conn == C_SYNC_TARGET)
			mod_timer(&mdev->resync_timer, jiffies);
	}

	if ((os.conn == C_SYNC_TARGET  || os.conn == C_SYNC_SOURCE) &&
	    (ns.conn == C_PAUSED_SYNC_T || ns.conn == C_PAUSED_SYNC_S)) {
		dev_info(DEV, "Resync suspended\n");
		mdev->rs_mark_time[mdev->rs_last_mark] = jiffies;
	}

	if (os.conn == C_CONNECTED &&
	    (ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T)) {
		unsigned long now = jiffies;
		int i;

		set_ov_position(mdev, ns.conn);
		mdev->rs_start = now;
		mdev->rs_last_events = 0;
		mdev->rs_last_sect_ev = 0;
		mdev->ov_last_oos_size = 0;
		mdev->ov_last_oos_start = 0;

		for (i = 0; i < DRBD_SYNC_MARKS; i++) {
			mdev->rs_mark_left[i] = mdev->ov_left;
			mdev->rs_mark_time[i] = now;
		}

		drbd_rs_controller_reset(mdev);

		if (ns.conn == C_VERIFY_S) {
			dev_info(DEV, "Starting Online Verify from sector %llu\n",
					(unsigned long long)mdev->ov_position);
			mod_timer(&mdev->resync_timer, jiffies);
		}
	}

	if (get_ldev(mdev)) {
		u32 mdf = mdev->ldev->md.flags & ~(MDF_CONSISTENT|MDF_PRIMARY_IND|
						 MDF_CONNECTED_IND|MDF_WAS_UP_TO_DATE|
						 MDF_PEER_OUT_DATED|MDF_CRASHED_PRIMARY);

1058
		mdf &= ~MDF_AL_CLEAN;
1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094
		if (test_bit(CRASHED_PRIMARY, &mdev->flags))
			mdf |= MDF_CRASHED_PRIMARY;
		if (mdev->state.role == R_PRIMARY ||
		    (mdev->state.pdsk < D_INCONSISTENT && mdev->state.peer == R_PRIMARY))
			mdf |= MDF_PRIMARY_IND;
		if (mdev->state.conn > C_WF_REPORT_PARAMS)
			mdf |= MDF_CONNECTED_IND;
		if (mdev->state.disk > D_INCONSISTENT)
			mdf |= MDF_CONSISTENT;
		if (mdev->state.disk > D_OUTDATED)
			mdf |= MDF_WAS_UP_TO_DATE;
		if (mdev->state.pdsk <= D_OUTDATED && mdev->state.pdsk >= D_INCONSISTENT)
			mdf |= MDF_PEER_OUT_DATED;
		if (mdf != mdev->ldev->md.flags) {
			mdev->ldev->md.flags = mdf;
			drbd_md_mark_dirty(mdev);
		}
		if (os.disk < D_CONSISTENT && ns.disk >= D_CONSISTENT)
			drbd_set_ed_uuid(mdev, mdev->ldev->md.uuid[UI_CURRENT]);
		put_ldev(mdev);
	}

	/* Peer was forced D_UP_TO_DATE & R_PRIMARY, consider to resync */
	if (os.disk == D_INCONSISTENT && os.pdsk == D_INCONSISTENT &&
	    os.peer == R_SECONDARY && ns.peer == R_PRIMARY)
		set_bit(CONSIDER_RESYNC, &mdev->flags);

	/* Receiver should clean up itself */
	if (os.conn != C_DISCONNECTING && ns.conn == C_DISCONNECTING)
		drbd_thread_stop_nowait(&mdev->tconn->receiver);

	/* Now the receiver finished cleaning up itself, it should die */
	if (os.conn != C_STANDALONE && ns.conn == C_STANDALONE)
		drbd_thread_stop_nowait(&mdev->tconn->receiver);

	/* Upon network failure, we need to restart the receiver. */
1095
	if (os.conn > C_WF_CONNECTION &&
1096 1097 1098 1099 1100 1101 1102
	    ns.conn <= C_TEAR_DOWN && ns.conn >= C_TIMEOUT)
		drbd_thread_restart_nowait(&mdev->tconn->receiver);

	/* Resume AL writing if we get a connection */
	if (os.conn < C_CONNECTED && ns.conn >= C_CONNECTED)
		drbd_resume_al(mdev);

1103 1104 1105 1106 1107 1108 1109
	/* remember last attach time so request_timer_fn() won't
	 * kill newly established sessions while we are still trying to thaw
	 * previously frozen IO */
	if ((os.disk == D_ATTACHING || os.disk == D_NEGOTIATING) &&
	    ns.disk > D_NEGOTIATING)
		mdev->last_reattach_jif = jiffies;

1110 1111 1112 1113 1114 1115
	ascw = kmalloc(sizeof(*ascw), GFP_ATOMIC);
	if (ascw) {
		ascw->os = os;
		ascw->ns = ns;
		ascw->flags = flags;
		ascw->w.cb = w_after_state_ch;
1116
		ascw->w.mdev = mdev;
1117
		ascw->done = done;
1118
		drbd_queue_work(&mdev->tconn->sender_work, &ascw->w);
1119
	} else {
1120
		dev_err(DEV, "Could not kmalloc an ascw\n");
1121 1122 1123 1124 1125
	}

	return rv;
}

1126
static int w_after_state_ch(struct drbd_work *w, int unused)
1127 1128 1129
{
	struct after_state_chg_work *ascw =
		container_of(w, struct after_state_chg_work, w);
1130
	struct drbd_conf *mdev = w->mdev;
1131 1132 1133 1134 1135 1136 1137 1138

	after_state_ch(mdev, ascw->os, ascw->ns, ascw->flags);
	if (ascw->flags & CS_WAIT_COMPLETE) {
		D_ASSERT(ascw->done != NULL);
		complete(ascw->done);
	}
	kfree(ascw);

1139
	return 0;
1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189
}

static void abw_start_sync(struct drbd_conf *mdev, int rv)
{
	if (rv) {
		dev_err(DEV, "Writing the bitmap failed not starting resync.\n");
		_drbd_request_state(mdev, NS(conn, C_CONNECTED), CS_VERBOSE);
		return;
	}

	switch (mdev->state.conn) {
	case C_STARTING_SYNC_T:
		_drbd_request_state(mdev, NS(conn, C_WF_SYNC_UUID), CS_VERBOSE);
		break;
	case C_STARTING_SYNC_S:
		drbd_start_resync(mdev, C_SYNC_SOURCE);
		break;
	}
}

int drbd_bitmap_io_from_worker(struct drbd_conf *mdev,
		int (*io_fn)(struct drbd_conf *),
		char *why, enum bm_flag flags)
{
	int rv;

	D_ASSERT(current == mdev->tconn->worker.task);

	/* open coded non-blocking drbd_suspend_io(mdev); */
	set_bit(SUSPEND_IO, &mdev->flags);

	drbd_bm_lock(mdev, why, flags);
	rv = io_fn(mdev);
	drbd_bm_unlock(mdev);

	drbd_resume_io(mdev);

	return rv;
}

/**
 * after_state_ch() - Perform after state change actions that may sleep
 * @mdev:	DRBD device.
 * @os:		old state.
 * @ns:		new state.
 * @flags:	Flags
 */
static void after_state_ch(struct drbd_conf *mdev, union drbd_state os,
			   union drbd_state ns, enum chg_state_flags flags)
{
1190 1191 1192 1193 1194
	struct sib_info sib;

	sib.sib_reason = SIB_STATE_CHANGE;
	sib.os = os;
	sib.ns = ns;
1195 1196 1197 1198 1199 1200 1201 1202

	if (os.conn != C_CONNECTED && ns.conn == C_CONNECTED) {
		clear_bit(CRASHED_PRIMARY, &mdev->flags);
		if (mdev->p_uuid)
			mdev->p_uuid[UI_FLAGS] &= ~((u64)2);
	}

	/* Inform userspace about the change... */
1203
	drbd_bcast_event(mdev, &sib);
1204 1205 1206 1207 1208 1209 1210 1211 1212

	if (!(os.role == R_PRIMARY && os.disk < D_UP_TO_DATE && os.pdsk < D_UP_TO_DATE) &&
	    (ns.role == R_PRIMARY && ns.disk < D_UP_TO_DATE && ns.pdsk < D_UP_TO_DATE))
		drbd_khelper(mdev, "pri-on-incon-degr");

	/* Here we have the actions that are performed after a
	   state change. This function might sleep */

	if (ns.susp_nod) {
1213
		struct drbd_tconn *tconn = mdev->tconn;
1214 1215
		enum drbd_req_event what = NOTHING;

1216 1217
		spin_lock_irq(&tconn->req_lock);
		if (os.conn < C_CONNECTED && conn_lowest_conn(tconn) >= C_CONNECTED)
1218 1219
			what = RESEND;

1220
		if ((os.disk == D_ATTACHING || os.disk == D_NEGOTIATING) &&
1221
		    conn_lowest_disk(tconn) > D_NEGOTIATING)
1222 1223
			what = RESTART_FROZEN_DISK_IO;

1224 1225 1226 1227 1228 1229
		if (tconn->susp_nod && what != NOTHING) {
			_tl_restart(tconn, what);
			_conn_request_state(tconn,
					    (union drbd_state) { { .susp_nod = 1 } },
					    (union drbd_state) { { .susp_nod = 0 } },
					    CS_VERBOSE);
1230
		}
1231
		spin_unlock_irq(&tconn->req_lock);
1232 1233
	}

1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255
	if (ns.susp_fen) {
		struct drbd_tconn *tconn = mdev->tconn;

		spin_lock_irq(&tconn->req_lock);
		if (tconn->susp_fen && conn_lowest_conn(tconn) >= C_CONNECTED) {
			/* case2: The connection was established again: */
			struct drbd_conf *odev;
			int vnr;

			rcu_read_lock();
			idr_for_each_entry(&tconn->volumes, odev, vnr)
				clear_bit(NEW_CUR_UUID, &odev->flags);
			rcu_read_unlock();
			_tl_restart(tconn, RESEND);
			_conn_request_state(tconn,
					    (union drbd_state) { { .susp_fen = 1 } },
					    (union drbd_state) { { .susp_fen = 0 } },
					    CS_VERBOSE);
		}
		spin_unlock_irq(&tconn->req_lock);
	}

1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267
	/* Became sync source.  With protocol >= 96, we still need to send out
	 * the sync uuid now. Need to do that before any drbd_send_state, or
	 * the other side may go "paused sync" before receiving the sync uuids,
	 * which is unexpected. */
	if ((os.conn != C_SYNC_SOURCE && os.conn != C_PAUSED_SYNC_S) &&
	    (ns.conn == C_SYNC_SOURCE || ns.conn == C_PAUSED_SYNC_S) &&
	    mdev->tconn->agreed_pro_version >= 96 && get_ldev(mdev)) {
		drbd_gen_and_send_sync_uuid(mdev);
		put_ldev(mdev);
	}

	/* Do not change the order of the if above and the two below... */
1268 1269
	if (os.pdsk == D_DISKLESS &&
	    ns.pdsk > D_DISKLESS && ns.pdsk != D_UNKNOWN) {      /* attach on the peer */
1270 1271 1272 1273 1274 1275 1276
		/* we probably will start a resync soon.
		 * make sure those things are properly reset. */
		mdev->rs_total = 0;
		mdev->rs_failed = 0;
		atomic_set(&mdev->rs_pending_cnt, 0);
		drbd_rs_cancel_all(mdev);

1277
		drbd_send_uuids(mdev);
1278
		drbd_send_state(mdev, ns);
1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298
	}
	/* No point in queuing send_bitmap if we don't have a connection
	 * anymore, so check also the _current_ state, not only the new state
	 * at the time this work was queued. */
	if (os.conn != C_WF_BITMAP_S && ns.conn == C_WF_BITMAP_S &&
	    mdev->state.conn == C_WF_BITMAP_S)
		drbd_queue_bitmap_io(mdev, &drbd_send_bitmap, NULL,
				"send_bitmap (WFBitMapS)",
				BM_LOCKED_TEST_ALLOWED);

	/* Lost contact to peer's copy of the data */
	if ((os.pdsk >= D_INCONSISTENT &&
	     os.pdsk != D_UNKNOWN &&
	     os.pdsk != D_OUTDATED)
	&&  (ns.pdsk < D_INCONSISTENT ||
	     ns.pdsk == D_UNKNOWN ||
	     ns.pdsk == D_OUTDATED)) {
		if (get_ldev(mdev)) {
			if ((ns.role == R_PRIMARY || ns.peer == R_PRIMARY) &&
			    mdev->ldev->md.uuid[UI_BITMAP] == 0 && ns.disk >= D_UP_TO_DATE) {
1299
				if (drbd_suspended(mdev)) {
1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310
					set_bit(NEW_CUR_UUID, &mdev->flags);
				} else {
					drbd_uuid_new_current(mdev);
					drbd_send_uuids(mdev);
				}
			}
			put_ldev(mdev);
		}
	}

	if (ns.pdsk < D_INCONSISTENT && get_ldev(mdev)) {
1311 1312 1313 1314 1315
		if (os.peer == R_SECONDARY && ns.peer == R_PRIMARY &&
		    mdev->ldev->md.uuid[UI_BITMAP] == 0 && ns.disk >= D_UP_TO_DATE) {
			drbd_uuid_new_current(mdev);
			drbd_send_uuids(mdev);
		}
1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342
		/* D_DISKLESS Peer becomes secondary */
		if (os.peer == R_PRIMARY && ns.peer == R_SECONDARY)
			/* We may still be Primary ourselves.
			 * No harm done if the bitmap still changes,
			 * redirtied pages will follow later. */
			drbd_bitmap_io_from_worker(mdev, &drbd_bm_write,
				"demote diskless peer", BM_LOCKED_SET_ALLOWED);
		put_ldev(mdev);
	}

	/* Write out all changed bits on demote.
	 * Though, no need to da that just yet
	 * if there is a resync going on still */
	if (os.role == R_PRIMARY && ns.role == R_SECONDARY &&
		mdev->state.conn <= C_CONNECTED && get_ldev(mdev)) {
		/* No changes to the bitmap expected this time, so assert that,
		 * even though no harm was done if it did change. */
		drbd_bitmap_io_from_worker(mdev, &drbd_bm_write,
				"demote", BM_LOCKED_TEST_ALLOWED);
		put_ldev(mdev);
	}

	/* Last part of the attaching process ... */
	if (ns.conn >= C_CONNECTED &&
	    os.disk == D_ATTACHING && ns.disk == D_NEGOTIATING) {
		drbd_send_sizes(mdev, 0, 0);  /* to start sync... */
		drbd_send_uuids(mdev);
1343
		drbd_send_state(mdev, ns);
1344 1345 1346 1347 1348 1349
	}

	/* We want to pause/continue resync, tell peer. */
	if (ns.conn >= C_CONNECTED &&
	     ((os.aftr_isp != ns.aftr_isp) ||
	      (os.user_isp != ns.user_isp)))
1350
		drbd_send_state(mdev, ns);
1351 1352 1353 1354 1355 1356 1357 1358 1359

	/* In case one of the isp bits got set, suspend other devices. */
	if ((!os.aftr_isp && !os.peer_isp && !os.user_isp) &&
	    (ns.aftr_isp || ns.peer_isp || ns.user_isp))
		suspend_other_sg(mdev);

	/* Make sure the peer gets informed about eventual state
	   changes (ISP bits) while we were in WFReportParams. */
	if (os.conn == C_WF_REPORT_PARAMS && ns.conn >= C_CONNECTED)
1360
		drbd_send_state(mdev, ns);
1361 1362

	if (os.conn != C_AHEAD && ns.conn == C_AHEAD)
1363
		drbd_send_state(mdev, ns);
1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382

	/* We are in the progress to start a full sync... */
	if ((os.conn != C_STARTING_SYNC_T && ns.conn == C_STARTING_SYNC_T) ||
	    (os.conn != C_STARTING_SYNC_S && ns.conn == C_STARTING_SYNC_S))
		/* no other bitmap changes expected during this phase */
		drbd_queue_bitmap_io(mdev,
			&drbd_bmio_set_n_write, &abw_start_sync,
			"set_n_write from StartingSync", BM_LOCKED_TEST_ALLOWED);

	/* We are invalidating our self... */
	if (os.conn < C_CONNECTED && ns.conn < C_CONNECTED &&
	    os.disk > D_INCONSISTENT && ns.disk == D_INCONSISTENT)
		/* other bitmap operation expected during this phase */
		drbd_queue_bitmap_io(mdev, &drbd_bmio_set_n_write, NULL,
			"set_n_write from invalidate", BM_LOCKED_MASK);

	/* first half of local IO error, failure to attach,
	 * or administrative detach */
	if (os.disk != D_FAILED && ns.disk == D_FAILED) {
1383 1384
		enum drbd_io_error_p eh = EP_PASS_ON;
		int was_io_error = 0;
1385
		/* corresponding get_ldev was in __drbd_set_state, to serialize
1386 1387 1388 1389 1390 1391 1392 1393 1394 1395
		 * our cleanup here with the transition to D_DISKLESS.
		 * But is is still not save to dreference ldev here, since
		 * we might come from an failed Attach before ldev was set. */
		if (mdev->ldev) {
			rcu_read_lock();
			eh = rcu_dereference(mdev->ldev->disk_conf)->on_io_error;
			rcu_read_unlock();

			was_io_error = test_and_clear_bit(WAS_IO_ERROR, &mdev->flags);

1396 1397 1398
			if (was_io_error && eh == EP_CALL_HELPER)
				drbd_khelper(mdev, "local-io-error");

1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413
			/* Immediately allow completion of all application IO,
			 * that waits for completion from the local disk,
			 * if this was a force-detach due to disk_timeout
			 * or administrator request (drbdsetup detach --force).
			 * Do NOT abort otherwise.
			 * Aborting local requests may cause serious problems,
			 * if requests are completed to upper layers already,
			 * and then later the already submitted local bio completes.
			 * This can cause DMA into former bio pages that meanwhile
			 * have been re-used for other things.
			 * So aborting local requests may cause crashes,
			 * or even worse, silent data corruption.
			 */
			if (test_and_clear_bit(FORCE_DETACH, &mdev->flags))
				tl_abort_disk_io(mdev);
1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432

			/* current state still has to be D_FAILED,
			 * there is only one way out: to D_DISKLESS,
			 * and that may only happen after our put_ldev below. */
			if (mdev->state.disk != D_FAILED)
				dev_err(DEV,
					"ASSERT FAILED: disk is %s during detach\n",
					drbd_disk_str(mdev->state.disk));

			if (ns.conn >= C_CONNECTED)
				drbd_send_state(mdev, ns);

			drbd_rs_cancel_all(mdev);

			/* In case we want to get something to stable storage still,
			 * this may be the last chance.
			 * Following put_ldev may transition to D_DISKLESS. */
			drbd_md_sync(mdev);
		}
1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446
		put_ldev(mdev);
	}

        /* second half of local IO error, failure to attach,
         * or administrative detach,
         * after local_cnt references have reached zero again */
        if (os.disk != D_DISKLESS && ns.disk == D_DISKLESS) {
                /* We must still be diskless,
                 * re-attach has to be serialized with this! */
                if (mdev->state.disk != D_DISKLESS)
                        dev_err(DEV,
                                "ASSERT FAILED: disk is %s while going diskless\n",
                                drbd_disk_str(mdev->state.disk));

1447 1448
		if (ns.conn >= C_CONNECTED)
			drbd_send_state(mdev, ns);
1449 1450 1451 1452 1453 1454
		/* corresponding get_ldev in __drbd_set_state
		 * this may finally trigger drbd_ldev_destroy. */
		put_ldev(mdev);
	}

	/* Notify peer that I had a local IO error, and did not detached.. */
1455
	if (os.disk == D_UP_TO_DATE && ns.disk == D_INCONSISTENT && ns.conn >= C_CONNECTED)
1456
		drbd_send_state(mdev, ns);
1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473

	/* Disks got bigger while they were detached */
	if (ns.disk > D_NEGOTIATING && ns.pdsk > D_NEGOTIATING &&
	    test_and_clear_bit(RESYNC_AFTER_NEG, &mdev->flags)) {
		if (ns.conn == C_CONNECTED)
			resync_after_online_grow(mdev);
	}

	/* A resync finished or aborted, wake paused devices... */
	if ((os.conn > C_CONNECTED && ns.conn <= C_CONNECTED) ||
	    (os.peer_isp && !ns.peer_isp) ||
	    (os.user_isp && !ns.user_isp))
		resume_next_sg(mdev);

	/* sync target done with resync.  Explicitly notify peer, even though
	 * it should (at least for non-empty resyncs) already know itself. */
	if (os.disk < D_UP_TO_DATE && os.conn >= C_SYNC_SOURCE && ns.conn == C_CONNECTED)
1474
		drbd_send_state(mdev, ns);
1475

1476 1477 1478 1479 1480 1481 1482
	/* Verify finished, or reached stop sector.  Peer did not know about
	 * the stop sector, and we may even have changed the stop sector during
	 * verify to interrupt/stop early.  Send the new state. */
	if (os.conn == C_VERIFY_S && ns.conn == C_CONNECTED
	&& verify_can_do_stop_sector(mdev))
		drbd_send_state(mdev, ns);

1483 1484 1485 1486 1487 1488 1489 1490
	/* This triggers bitmap writeout of potentially still unwritten pages
	 * if the resync finished cleanly, or aborted because of peer disk
	 * failure, or because of connection loss.
	 * For resync aborted because of local disk failure, we cannot do
	 * any bitmap writeout anymore.
	 * No harm done if some bits change during this phase.
	 */
	if (os.conn > C_CONNECTED && ns.conn <= C_CONNECTED && get_ldev(mdev)) {
1491 1492
		drbd_queue_bitmap_io(mdev, &drbd_bm_write_copy_pages, NULL,
			"write from resync_finished", BM_LOCKED_CHANGE_ALLOWED);
1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505
		put_ldev(mdev);
	}

	if (ns.disk == D_DISKLESS &&
	    ns.conn == C_STANDALONE &&
	    ns.role == R_SECONDARY) {
		if (os.aftr_isp != ns.aftr_isp)
			resume_next_sg(mdev);
	}

	drbd_md_sync(mdev);
}

1506 1507 1508
struct after_conn_state_chg_work {
	struct drbd_work w;
	enum drbd_conns oc;
1509
	union drbd_state ns_min;
1510
	union drbd_state ns_max; /* new, max state, over all mdevs */
1511 1512 1513
	enum chg_state_flags flags;
};

1514
static int w_after_conn_state_ch(struct drbd_work *w, int unused)
1515 1516 1517 1518 1519
{
	struct after_conn_state_chg_work *acscw =
		container_of(w, struct after_conn_state_chg_work, w);
	struct drbd_tconn *tconn = w->tconn;
	enum drbd_conns oc = acscw->oc;
1520
	union drbd_state ns_max = acscw->ns_max;
1521 1522
	struct drbd_conf *mdev;
	int vnr;
1523 1524 1525

	kfree(acscw);

1526
	/* Upon network configuration, we need to start the receiver */
1527
	if (oc == C_STANDALONE && ns_max.conn == C_UNCONNECTED)
1528 1529
		drbd_thread_start(&tconn->receiver);

1530 1531 1532
	if (oc == C_DISCONNECTING && ns_max.conn == C_STANDALONE) {
		struct net_conf *old_conf;

1533
		mutex_lock(&tconn->conf_update);
1534
		old_conf = tconn->net_conf;
1535 1536
		tconn->my_addr_len = 0;
		tconn->peer_addr_len = 0;
1537 1538
		rcu_assign_pointer(tconn->net_conf, NULL);
		conn_free_crypto(tconn);
1539
		mutex_unlock(&tconn->conf_update);
1540 1541 1542 1543 1544

		synchronize_rcu();
		kfree(old_conf);
	}

1545 1546 1547
	if (ns_max.susp_fen) {
		/* case1: The outdate peer handler is successful: */
		if (ns_max.pdsk <= D_OUTDATED) {
1548
			rcu_read_lock();
1549 1550 1551 1552 1553 1554
			idr_for_each_entry(&tconn->volumes, mdev, vnr) {
				if (test_bit(NEW_CUR_UUID, &mdev->flags)) {
					drbd_uuid_new_current(mdev);
					clear_bit(NEW_CUR_UUID, &mdev->flags);
				}
			}
1555
			rcu_read_unlock();
1556 1557 1558 1559 1560 1561 1562
			spin_lock_irq(&tconn->req_lock);
			_tl_restart(tconn, CONNECTION_LOST_WHILE_PENDING);
			_conn_request_state(tconn,
					    (union drbd_state) { { .susp_fen = 1 } },
					    (union drbd_state) { { .susp_fen = 0 } },
					    CS_VERBOSE);
			spin_unlock_irq(&tconn->req_lock);
1563 1564
		}
	}
1565
	kref_put(&tconn->kref, &conn_destroy);
1566
	return 0;
1567 1568
}

1569
void conn_old_common_state(struct drbd_tconn *tconn, union drbd_state *pcs, enum chg_state_flags *pf)
1570
{
1571
	enum chg_state_flags flags = ~0;
1572
	struct drbd_conf *mdev;
1573
	int vnr, first_vol = 1;
1574 1575 1576 1577 1578 1579 1580
	union drbd_dev_state os, cs = {
		{ .role = R_SECONDARY,
		  .peer = R_UNKNOWN,
		  .conn = tconn->cstate,
		  .disk = D_DISKLESS,
		  .pdsk = D_UNKNOWN,
		} };
1581

1582
	rcu_read_lock();
1583 1584 1585
	idr_for_each_entry(&tconn->volumes, mdev, vnr) {
		os = mdev->state;

1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599
		if (first_vol) {
			cs = os;
			first_vol = 0;
			continue;
		}

		if (cs.role != os.role)
			flags &= ~CS_DC_ROLE;

		if (cs.peer != os.peer)
			flags &= ~CS_DC_PEER;

		if (cs.conn != os.conn)
			flags &= ~CS_DC_CONN;
1600

1601 1602
		if (cs.disk != os.disk)
			flags &= ~CS_DC_DISK;
1603

1604 1605 1606
		if (cs.pdsk != os.pdsk)
			flags &= ~CS_DC_PDSK;
	}
1607
	rcu_read_unlock();
1608 1609 1610

	*pf |= CS_DC_MASK;
	*pf &= flags;
1611
	(*pcs).i = cs.i;
1612
}
1613

1614 1615
static enum drbd_state_rv
conn_is_valid_transition(struct drbd_tconn *tconn, union drbd_state mask, union drbd_state val,
1616
			 enum chg_state_flags flags)
1617
{
1618
	enum drbd_state_rv rv = SS_SUCCESS;
1619
	union drbd_state ns, os;
1620 1621
	struct drbd_conf *mdev;
	int vnr;
1622

1623
	rcu_read_lock();
1624
	idr_for_each_entry(&tconn->volumes, mdev, vnr) {
1625
		os = drbd_read_state(mdev);
1626 1627
		ns = sanitize_state(mdev, apply_mask_val(os, mask, val), NULL);

1628 1629 1630
		if (flags & CS_IGN_OUTD_FAIL && ns.disk == D_OUTDATED && os.disk < D_OUTDATED)
			ns.disk = os.disk;

1631 1632
		if (ns.i == os.i)
			continue;
1633

1634 1635 1636 1637 1638 1639 1640 1641
		rv = is_valid_transition(os, ns);
		if (rv < SS_SUCCESS)
			break;

		if (!(flags & CS_HARD)) {
			rv = is_valid_state(mdev, ns);
			if (rv < SS_SUCCESS) {
				if (is_valid_state(mdev, os) == rv)
1642
					rv = is_valid_soft_transition(os, ns, tconn);
1643
			} else
1644
				rv = is_valid_soft_transition(os, ns, tconn);
1645 1646 1647
		}
		if (rv < SS_SUCCESS)
			break;
1648
	}
1649
	rcu_read_unlock();
1650

1651 1652 1653 1654
	if (rv < SS_SUCCESS && flags & CS_VERBOSE)
		print_st_err(mdev, os, ns, rv);

	return rv;
1655 1656
}

1657
void
1658
conn_set_state(struct drbd_tconn *tconn, union drbd_state mask, union drbd_state val,
1659
	       union drbd_state *pns_min, union drbd_state *pns_max, enum chg_state_flags flags)
1660
{
1661
	union drbd_state ns, os, ns_max = { };
1662 1663 1664
	union drbd_state ns_min = {
		{ .role = R_MASK,
		  .peer = R_MASK,
1665
		  .conn = val.conn,
1666 1667 1668
		  .disk = D_MASK,
		  .pdsk = D_MASK
		} };
1669
	struct drbd_conf *mdev;
1670
	enum drbd_state_rv rv;
1671
	int vnr, number_of_volumes = 0;
1672

1673 1674 1675 1676 1677 1678 1679
	if (mask.conn == C_MASK) {
		/* remember last connect time so request_timer_fn() won't
		 * kill newly established sessions while we are still trying to thaw
		 * previously frozen IO */
		if (tconn->cstate != C_WF_REPORT_PARAMS && val.conn == C_WF_REPORT_PARAMS)
			tconn->last_reconnect_jif = jiffies;

1680
		tconn->cstate = val.conn;
1681
	}
1682

1683
	rcu_read_lock();
1684
	idr_for_each_entry(&tconn->volumes, mdev, vnr) {
1685
		number_of_volumes++;
1686
		os = drbd_read_state(mdev);
1687 1688
		ns = apply_mask_val(os, mask, val);
		ns = sanitize_state(mdev, ns, NULL);
1689

1690 1691 1692
		if (flags & CS_IGN_OUTD_FAIL && ns.disk == D_OUTDATED && os.disk < D_OUTDATED)
			ns.disk = os.disk;

1693 1694 1695
		rv = __drbd_set_state(mdev, ns, flags, NULL);
		if (rv < SS_SUCCESS)
			BUG();
1696

1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708
		ns.i = mdev->state.i;
		ns_max.role = max_role(ns.role, ns_max.role);
		ns_max.peer = max_role(ns.peer, ns_max.peer);
		ns_max.conn = max_t(enum drbd_conns, ns.conn, ns_max.conn);
		ns_max.disk = max_t(enum drbd_disk_state, ns.disk, ns_max.disk);
		ns_max.pdsk = max_t(enum drbd_disk_state, ns.pdsk, ns_max.pdsk);

		ns_min.role = min_role(ns.role, ns_min.role);
		ns_min.peer = min_role(ns.peer, ns_min.peer);
		ns_min.conn = min_t(enum drbd_conns, ns.conn, ns_min.conn);
		ns_min.disk = min_t(enum drbd_disk_state, ns.disk, ns_min.disk);
		ns_min.pdsk = min_t(enum drbd_disk_state, ns.pdsk, ns_min.pdsk);
1709
	}
1710
	rcu_read_unlock();
1711

1712 1713 1714 1715 1716 1717 1718 1719 1720 1721
	if (number_of_volumes == 0) {
		ns_min = ns_max = (union drbd_state) { {
				.role = R_SECONDARY,
				.peer = R_UNKNOWN,
				.conn = val.conn,
				.disk = D_DISKLESS,
				.pdsk = D_UNKNOWN
			} };
	}

1722 1723 1724 1725 1726 1727
	ns_min.susp = ns_max.susp = tconn->susp;
	ns_min.susp_nod = ns_max.susp_nod = tconn->susp_nod;
	ns_min.susp_fen = ns_max.susp_fen = tconn->susp_fen;

	*pns_min = ns_min;
	*pns_max = ns_max;
1728 1729
}

1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743
static enum drbd_state_rv
_conn_rq_cond(struct drbd_tconn *tconn, union drbd_state mask, union drbd_state val)
{
	enum drbd_state_rv rv;

	if (test_and_clear_bit(CONN_WD_ST_CHG_OKAY, &tconn->flags))
		return SS_CW_SUCCESS;

	if (test_and_clear_bit(CONN_WD_ST_CHG_FAIL, &tconn->flags))
		return SS_CW_FAILED_BY_PEER;

	rv = tconn->cstate != C_WF_REPORT_PARAMS ? SS_CW_NO_NEED : SS_UNKNOWN_ERROR;

	if (rv == SS_UNKNOWN_ERROR)
1744
		rv = conn_is_valid_transition(tconn, mask, val, 0);
1745

1746 1747
	if (rv == SS_SUCCESS)
		rv = SS_UNKNOWN_ERROR; /* cont waiting, otherwise fail. */
1748 1749 1750 1751

	return rv;
}

1752 1753 1754 1755 1756 1757 1758
enum drbd_state_rv
_conn_request_state(struct drbd_tconn *tconn, union drbd_state mask, union drbd_state val,
		    enum chg_state_flags flags)
{
	enum drbd_state_rv rv = SS_SUCCESS;
	struct after_conn_state_chg_work *acscw;
	enum drbd_conns oc = tconn->cstate;
1759
	union drbd_state ns_max, ns_min, os;
1760
	bool have_mutex = false;
1761

1762 1763 1764 1765 1766
	if (mask.conn) {
		rv = is_valid_conn_transition(oc, val.conn);
		if (rv < SS_SUCCESS)
			goto abort;
	}
1767

1768
	rv = conn_is_valid_transition(tconn, mask, val, flags);
1769 1770 1771
	if (rv < SS_SUCCESS)
		goto abort;

1772 1773
	if (oc == C_WF_REPORT_PARAMS && val.conn == C_DISCONNECTING &&
	    !(flags & (CS_LOCAL_ONLY | CS_HARD))) {
1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802

		/* This will be a cluster-wide state change.
		 * Need to give up the spinlock, grab the mutex,
		 * then send the state change request, ... */
		spin_unlock_irq(&tconn->req_lock);
		mutex_lock(&tconn->cstate_mutex);
		have_mutex = true;

		set_bit(CONN_WD_ST_CHG_REQ, &tconn->flags);
		if (conn_send_state_req(tconn, mask, val)) {
			/* sending failed. */
			clear_bit(CONN_WD_ST_CHG_REQ, &tconn->flags);
			rv = SS_CW_FAILED_BY_PEER;
			/* need to re-aquire the spin lock, though */
			goto abort_unlocked;
		}

		if (val.conn == C_DISCONNECTING)
			set_bit(DISCONNECT_SENT, &tconn->flags);

		/* ... and re-aquire the spinlock.
		 * If _conn_rq_cond() returned >= SS_SUCCESS, we must call
		 * conn_set_state() within the same spinlock. */
		spin_lock_irq(&tconn->req_lock);
		wait_event_lock_irq(tconn->ping_wait,
				(rv = _conn_rq_cond(tconn, mask, val)),
				tconn->req_lock,
				);
		clear_bit(CONN_WD_ST_CHG_REQ, &tconn->flags);
1803 1804 1805 1806
		if (rv < SS_SUCCESS)
			goto abort;
	}

1807
	conn_old_common_state(tconn, &os, &flags);
1808
	flags |= CS_DC_SUSP;
1809
	conn_set_state(tconn, mask, val, &ns_min, &ns_max, flags);
1810
	conn_pr_state_change(tconn, os, ns_max, flags);
1811 1812 1813

	acscw = kmalloc(sizeof(*acscw), GFP_ATOMIC);
	if (acscw) {
1814
		acscw->oc = os.conn;
1815
		acscw->ns_min = ns_min;
1816
		acscw->ns_max = ns_max;
1817 1818
		acscw->flags = flags;
		acscw->w.cb = w_after_conn_state_ch;
1819
		kref_get(&tconn->kref);
1820
		acscw->w.tconn = tconn;
1821
		drbd_queue_work(&tconn->sender_work, &acscw->w);
1822 1823
	} else {
		conn_err(tconn, "Could not kmalloc an acscw\n");
1824
	}
1825

1826
 abort:
1827 1828 1829 1830 1831 1832 1833 1834 1835
	if (have_mutex) {
		/* mutex_unlock() "... must not be used in interrupt context.",
		 * so give up the spinlock, then re-aquire it */
		spin_unlock_irq(&tconn->req_lock);
 abort_unlocked:
		mutex_unlock(&tconn->cstate_mutex);
		spin_lock_irq(&tconn->req_lock);
	}
	if (rv < SS_SUCCESS && flags & CS_VERBOSE) {
1836 1837 1838 1839
		conn_err(tconn, "State change failed: %s\n", drbd_set_st_err_str(rv));
		conn_err(tconn, " state = { cs:%s }\n", drbd_conn_str(oc));
		conn_err(tconn, "wanted = { cs:%s }\n", drbd_conn_str(val.conn));
	}
1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853
	return rv;
}

enum drbd_state_rv
conn_request_state(struct drbd_tconn *tconn, union drbd_state mask, union drbd_state val,
		   enum chg_state_flags flags)
{
	enum drbd_state_rv rv;

	spin_lock_irq(&tconn->req_lock);
	rv = _conn_request_state(tconn, mask, val, flags);
	spin_unlock_irq(&tconn->req_lock);

	return rv;
1854
}