scsi_lib.c 66.5 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9 10
/*
 *  scsi_lib.c Copyright (C) 1999 Eric Youngdale
 *
 *  SCSI queueing library.
 *      Initial versions: Eric Youngdale (eric@andante.org).
 *                        Based upon conversations with large numbers
 *                        of people at Linux Expo.
 */

#include <linux/bio.h>
11
#include <linux/bitops.h>
Linus Torvalds's avatar
Linus Torvalds committed
12 13 14
#include <linux/blkdev.h>
#include <linux/completion.h>
#include <linux/kernel.h>
15
#include <linux/export.h>
Linus Torvalds's avatar
Linus Torvalds committed
16 17 18 19 20
#include <linux/mempool.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/pci.h>
#include <linux/delay.h>
21
#include <linux/hardirq.h>
22
#include <linux/scatterlist.h>
Linus Torvalds's avatar
Linus Torvalds committed
23 24

#include <scsi/scsi.h>
25
#include <scsi/scsi_cmnd.h>
Linus Torvalds's avatar
Linus Torvalds committed
26 27 28 29 30 31 32 33 34 35
#include <scsi/scsi_dbg.h>
#include <scsi/scsi_device.h>
#include <scsi/scsi_driver.h>
#include <scsi/scsi_eh.h>
#include <scsi/scsi_host.h>

#include "scsi_priv.h"
#include "scsi_logging.h"


36
#define SG_MEMPOOL_NR		ARRAY_SIZE(scsi_sg_pools)
37
#define SG_MEMPOOL_SIZE		2
Linus Torvalds's avatar
Linus Torvalds committed
38 39 40

struct scsi_host_sg_pool {
	size_t		size;
41
	char		*name;
42
	struct kmem_cache	*slab;
Linus Torvalds's avatar
Linus Torvalds committed
43 44 45
	mempool_t	*pool;
};

46 47 48 49
#define SP(x) { x, "sgpool-" __stringify(x) }
#if (SCSI_MAX_SG_SEGMENTS < 32)
#error SCSI_MAX_SG_SEGMENTS is too small (must be 32 or greater)
#endif
50
static struct scsi_host_sg_pool scsi_sg_pools[] = {
Linus Torvalds's avatar
Linus Torvalds committed
51 52
	SP(8),
	SP(16),
53
#if (SCSI_MAX_SG_SEGMENTS > 32)
54
	SP(32),
55
#if (SCSI_MAX_SG_SEGMENTS > 64)
56 57
	SP(64),
#if (SCSI_MAX_SG_SEGMENTS > 128)
Linus Torvalds's avatar
Linus Torvalds committed
58
	SP(128),
59 60
#if (SCSI_MAX_SG_SEGMENTS > 256)
#error SCSI_MAX_SG_SEGMENTS is too large (256 MAX)
61 62 63
#endif
#endif
#endif
64 65
#endif
	SP(SCSI_MAX_SG_SEGMENTS)
66
};
Linus Torvalds's avatar
Linus Torvalds committed
67 68
#undef SP

69
struct kmem_cache *scsi_sdb_cache;
70

71 72 73 74 75 76 77
/*
 * When to reinvoke queueing after a resource shortage. It's 3 msecs to
 * not change behaviour from the previous unplug mechanism, experimentation
 * may prove this needs changing.
 */
#define SCSI_QUEUE_DELAY	3

78 79 80 81 82
/**
 * __scsi_queue_insert - private queue insertion
 * @cmd: The SCSI command being requeued
 * @reason:  The reason for the requeue
 * @unbusy: Whether the queue should be unbusied
Linus Torvalds's avatar
Linus Torvalds committed
83
 *
84 85 86 87 88
 * This is a private queue insertion.  The public interface
 * scsi_queue_insert() always assumes the queue should be unbusied
 * because it's always called before the completion.  This function is
 * for a requeue after completion, which should only occur in this
 * file.
Linus Torvalds's avatar
Linus Torvalds committed
89
 */
90
static void __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, int unbusy)
Linus Torvalds's avatar
Linus Torvalds committed
91 92 93
{
	struct Scsi_Host *host = cmd->device->host;
	struct scsi_device *device = cmd->device;
94
	struct scsi_target *starget = scsi_target(device);
95 96
	struct request_queue *q = device->request_queue;
	unsigned long flags;
Linus Torvalds's avatar
Linus Torvalds committed
97 98 99 100 101

	SCSI_LOG_MLQUEUE(1,
		 printk("Inserting command %p into mlqueue\n", cmd));

	/*
102
	 * Set the appropriate busy bit for the device/host.
Linus Torvalds's avatar
Linus Torvalds committed
103 104 105 106 107 108 109 110 111 112 113
	 *
	 * If the host/device isn't busy, assume that something actually
	 * completed, and that we should be able to queue a command now.
	 *
	 * Note that the prior mid-layer assumption that any host could
	 * always queue at least one command is now broken.  The mid-layer
	 * will implement a user specifiable stall (see
	 * scsi_host.max_host_blocked and scsi_device.max_device_blocked)
	 * if a command is requeued with no other commands outstanding
	 * either for the device or for the host.
	 */
114 115
	switch (reason) {
	case SCSI_MLQUEUE_HOST_BUSY:
Linus Torvalds's avatar
Linus Torvalds committed
116
		host->host_blocked = host->max_host_blocked;
117 118
		break;
	case SCSI_MLQUEUE_DEVICE_BUSY:
119
	case SCSI_MLQUEUE_EH_RETRY:
Linus Torvalds's avatar
Linus Torvalds committed
120
		device->device_blocked = device->max_device_blocked;
121 122 123 124 125
		break;
	case SCSI_MLQUEUE_TARGET_BUSY:
		starget->target_blocked = starget->max_target_blocked;
		break;
	}
Linus Torvalds's avatar
Linus Torvalds committed
126 127 128 129 130

	/*
	 * Decrement the counters, since these commands are no longer
	 * active on the host/device.
	 */
131 132
	if (unbusy)
		scsi_device_unbusy(device);
Linus Torvalds's avatar
Linus Torvalds committed
133 134

	/*
135
	 * Requeue this command.  It will go before all other commands
136 137 138
	 * that are already in the queue. Schedule requeue work under
	 * lock such that the kblockd_schedule_work() call happens
	 * before blk_cleanup_queue() finishes.
139
	 */
140
	cmd->result = 0;
141
	spin_lock_irqsave(q->queue_lock, flags);
142
	blk_requeue_request(q, cmd->request);
143
	kblockd_schedule_work(&device->requeue_work);
144
	spin_unlock_irqrestore(q->queue_lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
145 146
}

147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165
/*
 * Function:    scsi_queue_insert()
 *
 * Purpose:     Insert a command in the midlevel queue.
 *
 * Arguments:   cmd    - command that we are adding to queue.
 *              reason - why we are inserting command to queue.
 *
 * Lock status: Assumed that lock is not held upon entry.
 *
 * Returns:     Nothing.
 *
 * Notes:       We do this for one of two cases.  Either the host is busy
 *              and it cannot accept any more commands for the time being,
 *              or the device returned QUEUE_FULL and can accept no more
 *              commands.
 * Notes:       This could be called either from an interrupt context or a
 *              normal process context.
 */
166
void scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
167
{
168
	__scsi_queue_insert(cmd, reason, 1);
169
}
170
/**
171
 * scsi_execute - insert request and wait for the result
172 173 174 175 176 177 178 179
 * @sdev:	scsi device
 * @cmd:	scsi command
 * @data_direction: data direction
 * @buffer:	data buffer
 * @bufflen:	len of buffer
 * @sense:	optional sense buffer
 * @timeout:	request timeout in seconds
 * @retries:	number of times to retry request
180
 * @flags:	or into request flags;
181
 * @resid:	optional residual length
182
 *
183
 * returns the req->errors value which is the scsi_cmnd result
184
 * field.
185
 */
186 187
int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
		 int data_direction, void *buffer, unsigned bufflen,
188
		 unsigned char *sense, int timeout, int retries, u64 flags,
189
		 int *resid)
190 191 192 193 194 195
{
	struct request *req;
	int write = (data_direction == DMA_TO_DEVICE);
	int ret = DRIVER_ERROR << 24;

	req = blk_get_request(sdev->request_queue, write, __GFP_WAIT);
196 197
	if (!req)
		return ret;
198
	blk_rq_set_block_pc(req);
199 200 201 202 203 204 205 206 207

	if (bufflen &&	blk_rq_map_kern(sdev->request_queue, req,
					buffer, bufflen, __GFP_WAIT))
		goto out;

	req->cmd_len = COMMAND_SIZE(cmd[0]);
	memcpy(req->cmd, cmd, req->cmd_len);
	req->sense = sense;
	req->sense_len = 0;
208
	req->retries = retries;
209
	req->timeout = timeout;
210
	req->cmd_flags |= flags | REQ_QUIET | REQ_PREEMPT;
211 212 213 214 215 216

	/*
	 * head injection *required* here otherwise quiesce won't work
	 */
	blk_execute_rq(req->q, NULL, req, 1);

217 218 219 220 221 222
	/*
	 * Some devices (USB mass-storage in particular) may transfer
	 * garbage data together with a residue indicating that the data
	 * is invalid.  Prevent the garbage from being misinterpreted
	 * and prevent security leaks by zeroing out the excess data.
	 */
Tejun Heo's avatar
Tejun Heo committed
223 224
	if (unlikely(req->resid_len > 0 && req->resid_len <= bufflen))
		memset(buffer + (bufflen - req->resid_len), 0, req->resid_len);
225

226
	if (resid)
Tejun Heo's avatar
Tejun Heo committed
227
		*resid = req->resid_len;
228 229 230 231 232 233
	ret = req->errors;
 out:
	blk_put_request(req);

	return ret;
}
234
EXPORT_SYMBOL(scsi_execute);
235

236
int scsi_execute_req_flags(struct scsi_device *sdev, const unsigned char *cmd,
237
		     int data_direction, void *buffer, unsigned bufflen,
238
		     struct scsi_sense_hdr *sshdr, int timeout, int retries,
239
		     int *resid, u64 flags)
240 241
{
	char *sense = NULL;
242 243
	int result;
	
244
	if (sshdr) {
245
		sense = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_NOIO);
246 247 248
		if (!sense)
			return DRIVER_ERROR << 24;
	}
249
	result = scsi_execute(sdev, cmd, data_direction, buffer, bufflen,
250
			      sense, timeout, retries, flags, resid);
251
	if (sshdr)
252
		scsi_normalize_sense(sense, SCSI_SENSE_BUFFERSIZE, sshdr);
253 254 255 256

	kfree(sense);
	return result;
}
257
EXPORT_SYMBOL(scsi_execute_req_flags);
258

Linus Torvalds's avatar
Linus Torvalds committed
259 260 261 262 263 264 265 266 267 268 269
/*
 * Function:    scsi_init_cmd_errh()
 *
 * Purpose:     Initialize cmd fields related to error handling.
 *
 * Arguments:   cmd	- command that is ready to be queued.
 *
 * Notes:       This function has the job of initializing a number of
 *              fields related to error handling.   Typically this will
 *              be called once for each command, as required.
 */
270
static void scsi_init_cmd_errh(struct scsi_cmnd *cmd)
Linus Torvalds's avatar
Linus Torvalds committed
271 272
{
	cmd->serial_number = 0;
273
	scsi_set_resid(cmd, 0);
274
	memset(cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
Linus Torvalds's avatar
Linus Torvalds committed
275
	if (cmd->cmd_len == 0)
276
		cmd->cmd_len = scsi_command_size(cmd->cmnd);
Linus Torvalds's avatar
Linus Torvalds committed
277 278 279 280 281
}

void scsi_device_unbusy(struct scsi_device *sdev)
{
	struct Scsi_Host *shost = sdev->host;
282
	struct scsi_target *starget = scsi_target(sdev);
Linus Torvalds's avatar
Linus Torvalds committed
283 284 285 286
	unsigned long flags;

	spin_lock_irqsave(shost->host_lock, flags);
	shost->host_busy--;
287
	starget->target_busy--;
288
	if (unlikely(scsi_host_in_recovery(shost) &&
289
		     (shost->host_failed || shost->host_eh_scheduled)))
Linus Torvalds's avatar
Linus Torvalds committed
290 291
		scsi_eh_wakeup(shost);
	spin_unlock(shost->host_lock);
292
	spin_lock(sdev->request_queue->queue_lock);
Linus Torvalds's avatar
Linus Torvalds committed
293
	sdev->device_busy--;
294
	spin_unlock_irqrestore(sdev->request_queue->queue_lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342
}

/*
 * Called for single_lun devices on IO completion. Clear starget_sdev_user,
 * and call blk_run_queue for all the scsi_devices on the target -
 * including current_sdev first.
 *
 * Called with *no* scsi locks held.
 */
static void scsi_single_lun_run(struct scsi_device *current_sdev)
{
	struct Scsi_Host *shost = current_sdev->host;
	struct scsi_device *sdev, *tmp;
	struct scsi_target *starget = scsi_target(current_sdev);
	unsigned long flags;

	spin_lock_irqsave(shost->host_lock, flags);
	starget->starget_sdev_user = NULL;
	spin_unlock_irqrestore(shost->host_lock, flags);

	/*
	 * Call blk_run_queue for all LUNs on the target, starting with
	 * current_sdev. We race with others (to set starget_sdev_user),
	 * but in most cases, we will be first. Ideally, each LU on the
	 * target would get some limited time or requests on the target.
	 */
	blk_run_queue(current_sdev->request_queue);

	spin_lock_irqsave(shost->host_lock, flags);
	if (starget->starget_sdev_user)
		goto out;
	list_for_each_entry_safe(sdev, tmp, &starget->devices,
			same_target_siblings) {
		if (sdev == current_sdev)
			continue;
		if (scsi_device_get(sdev))
			continue;

		spin_unlock_irqrestore(shost->host_lock, flags);
		blk_run_queue(sdev->request_queue);
		spin_lock_irqsave(shost->host_lock, flags);
	
		scsi_device_put(sdev);
	}
 out:
	spin_unlock_irqrestore(shost->host_lock, flags);
}

343 344 345 346 347 348 349 350
static inline int scsi_device_is_busy(struct scsi_device *sdev)
{
	if (sdev->device_busy >= sdev->queue_depth || sdev->device_blocked)
		return 1;

	return 0;
}

351 352 353 354 355 356 357
static inline int scsi_target_is_busy(struct scsi_target *starget)
{
	return ((starget->can_queue > 0 &&
		 starget->target_busy >= starget->can_queue) ||
		 starget->target_blocked);
}

358 359 360 361 362 363 364 365 366
static inline int scsi_host_is_busy(struct Scsi_Host *shost)
{
	if ((shost->can_queue > 0 && shost->host_busy >= shost->can_queue) ||
	    shost->host_blocked || shost->host_self_blocked)
		return 1;

	return 0;
}

367
static void scsi_starved_list_run(struct Scsi_Host *shost)
Linus Torvalds's avatar
Linus Torvalds committed
368
{
369
	LIST_HEAD(starved_list);
370
	struct scsi_device *sdev;
Linus Torvalds's avatar
Linus Torvalds committed
371 372 373
	unsigned long flags;

	spin_lock_irqsave(shost->host_lock, flags);
374 375 376
	list_splice_init(&shost->starved_list, &starved_list);

	while (!list_empty(&starved_list)) {
377 378
		struct request_queue *slq;

Linus Torvalds's avatar
Linus Torvalds committed
379 380 381 382 383 384 385 386 387 388
		/*
		 * As long as shost is accepting commands and we have
		 * starved queues, call blk_run_queue. scsi_request_fn
		 * drops the queue_lock and can add us back to the
		 * starved_list.
		 *
		 * host_lock protects the starved_list and starved_entry.
		 * scsi_request_fn must get the host_lock before checking
		 * or modifying starved_list or starved_entry.
		 */
389
		if (scsi_host_is_busy(shost))
390 391
			break;

392 393 394
		sdev = list_entry(starved_list.next,
				  struct scsi_device, starved_entry);
		list_del_init(&sdev->starved_entry);
395 396 397 398 399 400
		if (scsi_target_is_busy(scsi_target(sdev))) {
			list_move_tail(&sdev->starved_entry,
				       &shost->starved_list);
			continue;
		}

401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419
		/*
		 * Once we drop the host lock, a racing scsi_remove_device()
		 * call may remove the sdev from the starved list and destroy
		 * it and the queue.  Mitigate by taking a reference to the
		 * queue and never touching the sdev again after we drop the
		 * host lock.  Note: if __scsi_remove_device() invokes
		 * blk_cleanup_queue() before the queue is run from this
		 * function then blk_run_queue() will return immediately since
		 * blk_cleanup_queue() marks the queue with QUEUE_FLAG_DYING.
		 */
		slq = sdev->request_queue;
		if (!blk_get_queue(slq))
			continue;
		spin_unlock_irqrestore(shost->host_lock, flags);

		blk_run_queue(slq);
		blk_put_queue(slq);

		spin_lock_irqsave(shost->host_lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
420
	}
421 422
	/* put any unprocessed entries back */
	list_splice(&starved_list, &shost->starved_list);
Linus Torvalds's avatar
Linus Torvalds committed
423
	spin_unlock_irqrestore(shost->host_lock, flags);
424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445
}

/*
 * Function:   scsi_run_queue()
 *
 * Purpose:    Select a proper request queue to serve next
 *
 * Arguments:  q       - last request's queue
 *
 * Returns:     Nothing
 *
 * Notes:      The previous command was completely finished, start
 *             a new one if possible.
 */
static void scsi_run_queue(struct request_queue *q)
{
	struct scsi_device *sdev = q->queuedata;

	if (scsi_target(sdev)->single_lun)
		scsi_single_lun_run(sdev);
	if (!list_empty(&sdev->host->starved_list))
		scsi_starved_list_run(sdev->host);
Linus Torvalds's avatar
Linus Torvalds committed
446 447 448 449

	blk_run_queue(q);
}

450 451 452 453 454 455 456 457 458 459
void scsi_requeue_run_queue(struct work_struct *work)
{
	struct scsi_device *sdev;
	struct request_queue *q;

	sdev = container_of(work, struct scsi_device, requeue_work);
	q = sdev->request_queue;
	scsi_run_queue(q);
}

Linus Torvalds's avatar
Linus Torvalds committed
460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475
/*
 * Function:	scsi_requeue_command()
 *
 * Purpose:	Handle post-processing of completed commands.
 *
 * Arguments:	q	- queue to operate on
 *		cmd	- command that may need to be requeued.
 *
 * Returns:	Nothing
 *
 * Notes:	After command completion, there may be blocks left
 *		over which weren't finished by the previous command
 *		this can be for a number of reasons - the main one is
 *		I/O errors in the middle of the request, in which case
 *		we need to request the blocks that come after the bad
 *		sector.
476
 * Notes:	Upon return, cmd is a stale pointer.
Linus Torvalds's avatar
Linus Torvalds committed
477 478 479
 */
static void scsi_requeue_command(struct request_queue *q, struct scsi_cmnd *cmd)
{
480
	struct scsi_device *sdev = cmd->device;
481
	struct request *req = cmd->request;
482 483 484
	unsigned long flags;

	spin_lock_irqsave(q->queue_lock, flags);
485 486 487
	blk_unprep_request(req);
	req->special = NULL;
	scsi_put_command(cmd);
488
	blk_requeue_request(q, req);
489
	spin_unlock_irqrestore(q->queue_lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
490 491

	scsi_run_queue(q);
492 493

	put_device(&sdev->sdev_gendev);
Linus Torvalds's avatar
Linus Torvalds committed
494 495 496 497
}

void scsi_next_command(struct scsi_cmnd *cmd)
{
498 499 500
	struct scsi_device *sdev = cmd->device;
	struct request_queue *q = sdev->request_queue;

Linus Torvalds's avatar
Linus Torvalds committed
501 502
	scsi_put_command(cmd);
	scsi_run_queue(q);
503 504

	put_device(&sdev->sdev_gendev);
Linus Torvalds's avatar
Linus Torvalds committed
505 506 507 508 509 510 511 512 513 514
}

void scsi_run_host_queues(struct Scsi_Host *shost)
{
	struct scsi_device *sdev;

	shost_for_each_device(sdev, shost)
		scsi_run_queue(sdev->request_queue);
}

515 516 517 518
static inline unsigned int scsi_sgtable_index(unsigned short nents)
{
	unsigned int index;

519 520 521
	BUG_ON(nents > SCSI_MAX_SG_SEGMENTS);

	if (nents <= 8)
522
		index = 0;
523 524
	else
		index = get_count_order(nents) - 3;
Linus Torvalds's avatar
Linus Torvalds committed
525

526 527 528
	return index;
}

529
static void scsi_sg_free(struct scatterlist *sgl, unsigned int nents)
530 531 532
{
	struct scsi_host_sg_pool *sgp;

533 534 535
	sgp = scsi_sg_pools + scsi_sgtable_index(nents);
	mempool_free(sgl, sgp->pool);
}
536

537 538 539
static struct scatterlist *scsi_sg_alloc(unsigned int nents, gfp_t gfp_mask)
{
	struct scsi_host_sg_pool *sgp;
540

541 542 543
	sgp = scsi_sg_pools + scsi_sgtable_index(nents);
	return mempool_alloc(sgp->pool, gfp_mask);
}
544

545 546
static int scsi_alloc_sgtable(struct scsi_data_buffer *sdb, int nents,
			      gfp_t gfp_mask)
547 548
{
	int ret;
549

550
	BUG_ON(!nents);
551

552 553
	ret = __sg_alloc_table(&sdb->table, nents, SCSI_MAX_SG_SEGMENTS,
			       gfp_mask, scsi_sg_alloc);
554
	if (unlikely(ret))
555
		__sg_free_table(&sdb->table, SCSI_MAX_SG_SEGMENTS,
556
				scsi_sg_free);
557

558
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
559 560
}

561
static void scsi_free_sgtable(struct scsi_data_buffer *sdb)
Linus Torvalds's avatar
Linus Torvalds committed
562
{
563
	__sg_free_table(&sdb->table, SCSI_MAX_SG_SEGMENTS, scsi_sg_free);
Linus Torvalds's avatar
Linus Torvalds committed
564 565 566 567 568
}

/*
 * Function:    scsi_release_buffers()
 *
569
 * Purpose:     Free resources allocate for a scsi_command.
Linus Torvalds's avatar
Linus Torvalds committed
570 571 572 573 574 575 576 577 578 579
 *
 * Arguments:   cmd	- command that we are bailing.
 *
 * Lock status: Assumed that no lock is held upon entry.
 *
 * Returns:     Nothing
 *
 * Notes:       In the event that an upper level driver rejects a
 *		command, we must release resources allocated during
 *		the __init_io() function.  Primarily this would involve
580
 *		the scatter-gather table.
Linus Torvalds's avatar
Linus Torvalds committed
581
 */
582
void scsi_release_buffers(struct scsi_cmnd *cmd)
Linus Torvalds's avatar
Linus Torvalds committed
583
{
584 585 586 587 588 589 590
	if (cmd->sdb.table.nents)
		scsi_free_sgtable(&cmd->sdb);

	memset(&cmd->sdb, 0, sizeof(cmd->sdb));

	if (scsi_prot_sg_count(cmd))
		scsi_free_sgtable(cmd->prot_sdb);
Linus Torvalds's avatar
Linus Torvalds committed
591
}
592
EXPORT_SYMBOL(scsi_release_buffers);
Linus Torvalds's avatar
Linus Torvalds committed
593

594 595 596 597 598 599 600 601 602
static void scsi_release_bidi_buffers(struct scsi_cmnd *cmd)
{
	struct scsi_data_buffer *bidi_sdb = cmd->request->next_rq->special;

	scsi_free_sgtable(bidi_sdb);
	kmem_cache_free(scsi_sdb_cache, bidi_sdb);
	cmd->request->next_rq->special = NULL;
}

603 604 605 606 607 608 609 610 611 612
/**
 * __scsi_error_from_host_byte - translate SCSI error code into errno
 * @cmd:	SCSI command (unused)
 * @result:	scsi error code
 *
 * Translate SCSI error code into standard UNIX errno.
 * Return values:
 * -ENOLINK	temporary transport failure
 * -EREMOTEIO	permanent target failure, do not retry
 * -EBADE	permanent nexus failure, retry on other path
613
 * -ENOSPC	No write space available
614
 * -ENODATA	Medium error
615 616
 * -EIO		unspecified I/O error
 */
617 618 619 620 621 622 623 624 625
static int __scsi_error_from_host_byte(struct scsi_cmnd *cmd, int result)
{
	int error = 0;

	switch(host_byte(result)) {
	case DID_TRANSPORT_FAILFAST:
		error = -ENOLINK;
		break;
	case DID_TARGET_FAILURE:
626
		set_host_byte(cmd, DID_OK);
627 628 629
		error = -EREMOTEIO;
		break;
	case DID_NEXUS_FAILURE:
630
		set_host_byte(cmd, DID_OK);
631 632
		error = -EBADE;
		break;
633 634 635 636
	case DID_ALLOC_FAILURE:
		set_host_byte(cmd, DID_OK);
		error = -ENOSPC;
		break;
637 638 639 640
	case DID_MEDIUM_ERROR:
		set_host_byte(cmd, DID_OK);
		error = -ENODATA;
		break;
641 642 643 644 645 646 647 648
	default:
		error = -EIO;
		break;
	}

	return error;
}

Linus Torvalds's avatar
Linus Torvalds committed
649 650 651 652 653 654 655 656 657 658 659
/*
 * Function:    scsi_io_completion()
 *
 * Purpose:     Completion processing for block device I/O requests.
 *
 * Arguments:   cmd   - command that is finished.
 *
 * Lock status: Assumed that no lock is held upon entry.
 *
 * Returns:     Nothing
 *
660 661 662
 * Notes:       We will finish off the specified number of sectors.  If we
 *		are done, the command block will be released and the queue
 *		function will be goosed.  If we are not done then we have to
663
 *		figure out what to do next:
Linus Torvalds's avatar
Linus Torvalds committed
664
 *
665 666 667 668 669
 *		a) We can call scsi_requeue_command().  The request
 *		   will be unprepared and put back on the queue.  Then
 *		   a new command will be created for it.  This should
 *		   be used if we made forward progress, or if we want
 *		   to switch from READ(10) to READ(6) for example.
Linus Torvalds's avatar
Linus Torvalds committed
670
 *
671
 *		b) We can call __scsi_queue_insert().  The request will
672 673 674 675 676
 *		   be put back on the queue and retried using the same
 *		   command as before, possibly after a delay.
 *
 *		c) We can call blk_end_request() with -EIO to fail
 *		   the remainder of the request.
Linus Torvalds's avatar
Linus Torvalds committed
677
 */
678
void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
Linus Torvalds's avatar
Linus Torvalds committed
679 680
{
	int result = cmd->result;
681
	struct request_queue *q = cmd->device->request_queue;
Linus Torvalds's avatar
Linus Torvalds committed
682
	struct request *req = cmd->request;
683
	int error = 0;
Linus Torvalds's avatar
Linus Torvalds committed
684 685 686
	struct scsi_sense_hdr sshdr;
	int sense_valid = 0;
	int sense_deferred = 0;
687 688 689
	enum {ACTION_FAIL, ACTION_REPREP, ACTION_RETRY,
	      ACTION_DELAYED_RETRY} action;
	char *description = NULL;
690
	unsigned long wait_for = (cmd->allowed + 1) * req->timeout;
Linus Torvalds's avatar
Linus Torvalds committed
691 692 693 694 695 696

	if (result) {
		sense_valid = scsi_command_normalize_sense(cmd, &sshdr);
		if (sense_valid)
			sense_deferred = scsi_sense_is_deferred(&sshdr);
	}
697

698
	if (req->cmd_type == REQ_TYPE_BLOCK_PC) { /* SG_IO ioctl from block level */
Linus Torvalds's avatar
Linus Torvalds committed
699 700 701 702 703 704 705 706 707 708 709 710
		if (result) {
			if (sense_valid && req->sense) {
				/*
				 * SG_IO wants current and deferred errors
				 */
				int len = 8 + cmd->sense_buffer[7];

				if (len > SCSI_SENSE_BUFFERSIZE)
					len = SCSI_SENSE_BUFFERSIZE;
				memcpy(req->sense, cmd->sense_buffer,  len);
				req->sense_len = len;
			}
711
			if (!sense_deferred)
712
				error = __scsi_error_from_host_byte(cmd, result);
713
		}
714 715 716 717
		/*
		 * __scsi_error_from_host_byte may have reset the host_byte
		 */
		req->errors = cmd->result;
718 719 720

		req->resid_len = scsi_get_resid(cmd);

721
		if (scsi_bidi_cmnd(cmd)) {
722 723 724 725 726 727
			/*
			 * Bidi commands Must be complete as a whole,
			 * both sides at once.
			 */
			req->next_rq->resid_len = scsi_in(cmd)->resid;

728
			scsi_release_buffers(cmd);
729
			scsi_release_bidi_buffers(cmd);
730

731 732 733
			blk_end_request_all(req, 0);

			scsi_next_command(cmd);
734 735
			return;
		}
736 737 738 739 740 741 742 743
	} else if (blk_rq_bytes(req) == 0 && result && !sense_deferred) {
		/*
		 * Certain non BLOCK_PC requests are commands that don't
		 * actually transfer anything (FLUSH), so cannot use
		 * good_bytes != blk_rq_bytes(req) as the signal for an error.
		 * This sets the error explicitly for the problem case.
		 */
		error = __scsi_error_from_host_byte(cmd, result);
Linus Torvalds's avatar
Linus Torvalds committed
744 745
	}

746 747
	/* no bidi support for !REQ_TYPE_BLOCK_PC yet */
	BUG_ON(blk_bidi_rq(req));
748

Linus Torvalds's avatar
Linus Torvalds committed
749 750 751 752
	/*
	 * Next deal with any sectors which we were able to correctly
	 * handle.
	 */
753
	SCSI_LOG_HLCOMPLETE(1, printk("%u sectors total, "
754
				      "%d bytes done.\n",
755
				      blk_rq_sectors(req), good_bytes));
756

757 758 759 760 761 762
	/*
	 * Recovered errors need reporting, but they're always treated
	 * as success, so fiddle the result code here.  For BLOCK_PC
	 * we already took a copy of the original into rq->errors which
	 * is what gets returned to the user
	 */
763 764 765 766 767 768 769 770
	if (sense_valid && (sshdr.sense_key == RECOVERED_ERROR)) {
		/* if ATA PASS-THROUGH INFORMATION AVAILABLE skip
		 * print since caller wants ATA registers. Only occurs on
		 * SCSI ATA PASS_THROUGH commands when CK_COND=1
		 */
		if ((sshdr.asc == 0x0) && (sshdr.ascq == 0x1d))
			;
		else if (!(req->cmd_flags & REQ_QUIET))
771 772 773 774 775 776 777
			scsi_print_sense("", cmd);
		result = 0;
		/* BLOCK_PC may have set error */
		error = 0;
	}

	/*
778 779 780
	 * special case: failed zero length commands always need to
	 * drop down into the retry code. Otherwise, if we finished
	 * all bytes in the request we are done now.
781
	 */
782 783
	if (!(blk_rq_bytes(req) == 0 && error) &&
	    !blk_end_request(req, error, good_bytes))
784 785 786 787 788 789 790 791 792 793 794 795 796
		goto next_command;

	/*
	 * Kill remainder if no retrys.
	 */
	if (error && scsi_noretry_cmd(cmd)) {
		blk_end_request_all(req, error);
		goto next_command;
	}

	/*
	 * If there had been no error, but we have leftover bytes in the
	 * requeues just queue the command up again.
797
	 */
798 799
	if (result == 0)
		goto requeue;
800

801
	error = __scsi_error_from_host_byte(cmd, result);
802

803 804 805 806 807 808 809
	if (host_byte(result) == DID_RESET) {
		/* Third party bus reset or reset for error recovery
		 * reasons.  Just retry the command and see what
		 * happens.
		 */
		action = ACTION_RETRY;
	} else if (sense_valid && !sense_deferred) {
Linus Torvalds's avatar
Linus Torvalds committed
810 811 812
		switch (sshdr.sense_key) {
		case UNIT_ATTENTION:
			if (cmd->device->removable) {
813
				/* Detected disc change.  Set a bit
Linus Torvalds's avatar
Linus Torvalds committed
814 815 816
				 * and quietly refuse further access.
				 */
				cmd->device->changed = 1;
817 818
				description = "Media Changed";
				action = ACTION_FAIL;
Linus Torvalds's avatar
Linus Torvalds committed
819
			} else {
820 821 822
				/* Must have been a power glitch, or a
				 * bus reset.  Could not have been a
				 * media change, so we just retry the
823
				 * command and see what happens.
824
				 */
825
				action = ACTION_RETRY;
Linus Torvalds's avatar
Linus Torvalds committed
826 827 828
			}
			break;
		case ILLEGAL_REQUEST:
829 830 831 832 833 834 835 836
			/* If we had an ILLEGAL REQUEST returned, then
			 * we may have performed an unsupported
			 * command.  The only thing this should be
			 * would be a ten byte read where only a six
			 * byte read was supported.  Also, on a system
			 * where READ CAPACITY failed, we may have
			 * read past the end of the disk.
			 */
837 838
			if ((cmd->device->use_10_for_rw &&
			    sshdr.asc == 0x20 && sshdr.ascq == 0x00) &&
Linus Torvalds's avatar
Linus Torvalds committed
839 840
			    (cmd->cmnd[0] == READ_10 ||
			     cmd->cmnd[0] == WRITE_10)) {
841
				/* This will issue a new 6-byte command. */
Linus Torvalds's avatar
Linus Torvalds committed
842
				cmd->device->use_10_for_rw = 0;
843
				action = ACTION_REPREP;
844 845 846 847
			} else if (sshdr.asc == 0x10) /* DIX */ {
				description = "Host Data Integrity Failure";
				action = ACTION_FAIL;
				error = -EILSEQ;
848
			/* INVALID COMMAND OPCODE or INVALID FIELD IN CDB */
849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865
			} else if (sshdr.asc == 0x20 || sshdr.asc == 0x24) {
				switch (cmd->cmnd[0]) {
				case UNMAP:
					description = "Discard failure";
					break;
				case WRITE_SAME:
				case WRITE_SAME_16:
					if (cmd->cmnd[1] & 0x8)
						description = "Discard failure";
					else
						description =
							"Write same failure";
					break;
				default:
					description = "Invalid command failure";
					break;
				}
866
				action = ACTION_FAIL;
867
				error = -EREMOTEIO;
868 869 870
			} else
				action = ACTION_FAIL;
			break;
871
		case ABORTED_COMMAND:
872
			action = ACTION_FAIL;
873
			if (sshdr.asc == 0x10) { /* DIF */
874 875
				description = "Target Data Integrity Failure";
				error = -EILSEQ;
876
			}
Linus Torvalds's avatar
Linus Torvalds committed
877 878
			break;
		case NOT_READY:
879
			/* If the device is in the process of becoming
880
			 * ready, or has a temporary blockage, retry.
Linus Torvalds's avatar
Linus Torvalds committed
881
			 */
882 883 884 885 886 887 888 889 890
			if (sshdr.asc == 0x04) {
				switch (sshdr.ascq) {
				case 0x01: /* becoming ready */
				case 0x04: /* format in progress */
				case 0x05: /* rebuild in progress */
				case 0x06: /* recalculation in progress */
				case 0x07: /* operation in progress */
				case 0x08: /* Long write in progress */
				case 0x09: /* self test in progress */
891
				case 0x14: /* space allocation in progress */
892
					action = ACTION_DELAYED_RETRY;
893
					break;
894 895 896 897
				default:
					description = "Device not ready";
					action = ACTION_FAIL;
					break;
898
				}
899 900 901
			} else {
				description = "Device not ready";
				action = ACTION_FAIL;
Linus Torvalds's avatar
Linus Torvalds committed
902
			}
903
			break;
Linus Torvalds's avatar
Linus Torvalds committed
904
		case VOLUME_OVERFLOW:
905
			/* See SSC3rXX or current. */
906 907
			action = ACTION_FAIL;
			break;
Linus Torvalds's avatar
Linus Torvalds committed
908
		default:
909 910
			description = "Unhandled sense code";
			action = ACTION_FAIL;
Linus Torvalds's avatar
Linus Torvalds committed
911 912
			break;
		}
913 914 915
	} else {
		description = "Unhandled error code";
		action = ACTION_FAIL;
916
	}
917

918 919 920 921 922 923
	if (action != ACTION_FAIL &&
	    time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) {
		action = ACTION_FAIL;
		description = "Command timed out";
	}

924 925 926
	switch (action) {
	case ACTION_FAIL:
		/* Give up and fail the remainder of the request */
927
		if (!(req->cmd_flags & REQ_QUIET)) {
928
			if (description)
929
				scmd_printk(KERN_INFO, cmd, "%s\n",
930
					    description);
931
			scsi_print_result(cmd);
932 933
			if (driver_byte(result) & DRIVER_SENSE)
				scsi_print_sense("", cmd);
934
			scsi_print_command(cmd);
935
		}
936 937 938
		if (!blk_end_request_err(req, error))
			goto next_command;
		/*FALLTHRU*/
939
	case ACTION_REPREP:
940
	requeue:
941 942 943
		/* Unprep the request and put it back at the head of the queue.
		 * A new command will be prepared and issued.
		 */
944
		scsi_release_buffers(cmd);
945 946 947 948
		scsi_requeue_command(q, cmd);
		break;
	case ACTION_RETRY:
		/* Retry the same command immediately */
949
		__scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY, 0);
950 951 952
		break;
	case ACTION_DELAYED_RETRY:
		/* Retry the same command after a delay */
953
		__scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY, 0);
954
		break;
Linus Torvalds's avatar
Linus Torvalds committed
955
	}
956 957 958 959 960
	return;

next_command:
	scsi_release_buffers(cmd);
	scsi_next_command(cmd);
Linus Torvalds's avatar
Linus Torvalds committed
961 962
}

963 964
static int scsi_init_sgtable(struct request *req, struct scsi_data_buffer *sdb,
			     gfp_t gfp_mask)
Linus Torvalds's avatar
Linus Torvalds committed
965
{
966
	int count;
Linus Torvalds's avatar
Linus Torvalds committed
967 968

	/*
969
	 * If sg table allocation fails, requeue request later.
Linus Torvalds's avatar
Linus Torvalds committed
970
	 */
971 972
	if (unlikely(scsi_alloc_sgtable(sdb, req->nr_phys_segments,
					gfp_mask))) {
Linus Torvalds's avatar
Linus Torvalds committed
973
		return BLKPREP_DEFER;
Alan Stern's avatar
Alan Stern committed
974
	}
Linus Torvalds's avatar
Linus Torvalds committed
975 976 977 978 979

	/* 
	 * Next, walk the list, and fill in the addresses and sizes of
	 * each segment.
	 */
980 981 982
	count = blk_rq_map_sg(req->q, req, sdb->table.sgl);
	BUG_ON(count > sdb->table.nents);
	sdb->table.nents = count;
983
	sdb->length = blk_rq_bytes(req);
984
	return BLKPREP_OK;
Linus Torvalds's avatar
Linus Torvalds committed
985
}
986 987 988 989 990 991 992 993 994 995 996 997 998 999

/*
 * Function:    scsi_init_io()
 *
 * Purpose:     SCSI I/O initialize function.
 *
 * Arguments:   cmd   - Command descriptor we wish to initialize
 *
 * Returns:     0 on success
 *		BLKPREP_DEFER if the failure is retryable
 *		BLKPREP_KILL if the failure is fatal
 */
int scsi_init_io(struct scsi_cmnd *cmd, gfp_t gfp_mask)
{
1000
	struct scsi_device *sdev = cmd->device;
1001 1002 1003
	struct request *rq = cmd->request;

	int error = scsi_init_sgtable(rq, &cmd->sdb, gfp_mask);
1004 1005 1006
	if (error)
		goto err_exit;

1007
	if (blk_bidi_rq(rq)) {
1008
		struct scsi_data_buffer *bidi_sdb = kmem_cache_zalloc(
1009
			scsi_sdb_cache, GFP_ATOMIC);
1010 1011 1012 1013 1014
		if (!bidi_sdb) {
			error = BLKPREP_DEFER;
			goto err_exit;
		}

1015 1016
		rq->next_rq->special = bidi_sdb;
		error = scsi_init_sgtable(rq->next_rq, bidi_sdb, GFP_ATOMIC);
1017 1018 1019 1020
		if (error)
			goto err_exit;
	}

1021
	if (blk_integrity_rq(rq)) {
1022 1023 1024 1025
		struct scsi_data_buffer *prot_sdb = cmd->prot_sdb;
		int ivecs, count;

		BUG_ON(prot_sdb == NULL);
1026
		ivecs = blk_rq_count_integrity_sg(rq->q, rq->bio);
1027 1028 1029 1030 1031 1032

		if (scsi_alloc_sgtable(prot_sdb, ivecs, gfp_mask)) {
			error = BLKPREP_DEFER;
			goto err_exit;
		}

1033
		count = blk_rq_map_integrity_sg(rq->q, rq->bio,
1034 1035
						prot_sdb->table.sgl);
		BUG_ON(unlikely(count > ivecs));
1036
		BUG_ON(unlikely(count > queue_max_integrity_segments(rq->q)));
1037 1038 1039 1040 1041

		cmd->prot_sdb = prot_sdb;
		cmd->prot_sdb->table.nents = count;
	}

1042 1043 1044 1045
	return BLKPREP_OK ;

err_exit:
	scsi_release_buffers(cmd);
1046
	cmd->request->special = NULL;
1047
	scsi_put_command(cmd);
1048
	put_device(&sdev->sdev_gendev);
1049 1050
	return error;
}
1051
EXPORT_SYMBOL(scsi_init_io);
Linus Torvalds's avatar
Linus Torvalds committed
1052

1053 1054 1055 1056 1057 1058
static struct scsi_cmnd *scsi_get_cmd_from_req(struct scsi_device *sdev,
		struct request *req)
{
	struct scsi_cmnd *cmd;

	if (!req->special) {
1059 1060 1061 1062
		/* Bail if we can't get a reference to the device */
		if (!get_device(&sdev->sdev_gendev))
			return NULL;

1063
		cmd = scsi_get_command(sdev, GFP_ATOMIC);
1064 1065
		if (unlikely(!cmd)) {
			put_device(&sdev->sdev_gendev);
1066
			return NULL;
1067
		}
1068 1069 1070 1071 1072 1073 1074 1075 1076
		req->special = cmd;
	} else {
		cmd = req->special;
	}

	/* pull a tag out of the request if we have one */
	cmd->tag = req->tag;
	cmd->request = req;

1077
	cmd->cmnd = req->cmd;
1078
	cmd->prot_op = SCSI_PROT_NORMAL;
1079

1080 1081 1082
	return cmd;
}

1083
int scsi_setup_blk_pc_cmnd(struct scsi_device *sdev, struct request *req)
James Bottomley's avatar
James Bottomley committed
1084
{
1085
	struct scsi_cmnd *cmd = req->special;
1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097

	/*
	 * BLOCK_PC requests may transfer data, in which case they must
	 * a bio attached to them.  Or they might contain a SCSI command
	 * that does not transfer data, in which case they may optionally
	 * submit a request without an attached bio.
	 */
	if (req->bio) {
		int ret;

		BUG_ON(!req->nr_phys_segments);

1098
		ret = scsi_init_io(cmd, GFP_ATOMIC);
1099 1100 1101
		if (unlikely(ret))
			return ret;
	} else {
1102
		BUG_ON(blk_rq_bytes(req));
1103

1104
		memset(&cmd->sdb, 0, sizeof(cmd->sdb));
1105
	}
James Bottomley's avatar
James Bottomley committed
1106 1107

	cmd->cmd_len = req->cmd_len;
1108
	if (!blk_rq_bytes(req))
James Bottomley's avatar
James Bottomley committed
1109 1110 1111 1112 1113 1114
		cmd->sc_data_direction = DMA_NONE;
	else if (rq_data_dir(req) == WRITE)
		cmd->sc_data_direction = DMA_TO_DEVICE;
	else
		cmd->sc_data_direction = DMA_FROM_DEVICE;
	
1115
	cmd->transfersize = blk_rq_bytes(req);
James Bottomley's avatar
James Bottomley committed
1116
	cmd->allowed = req->retries;
1117
	return BLKPREP_OK;
James Bottomley's avatar
James Bottomley committed
1118
}
1119
EXPORT_SYMBOL(scsi_setup_blk_pc_cmnd);
James Bottomley's avatar
James Bottomley committed
1120

1121 1122 1123 1124 1125
/*
 * Setup a REQ_TYPE_FS command.  These are simple read/write request
 * from filesystems that still need to be translated to SCSI CDBs from
 * the ULD.
 */
1126
int scsi_setup_fs_cmnd(struct scsi_device *sdev, struct request *req)
Linus Torvalds's avatar
Linus Torvalds committed
1127
{
1128
	struct scsi_cmnd *cmd = req->special;
1129 1130 1131

	if (unlikely(sdev->scsi_dh_data && sdev->scsi_dh_data->scsi_dh
			 && sdev->scsi_dh_data->scsi_dh->prep_fn)) {
1132
		int ret = sdev->scsi_dh_data->scsi_dh->prep_fn(sdev, req);
1133 1134 1135 1136
		if (ret != BLKPREP_OK)
			return ret;
	}

Linus Torvalds's avatar
Linus Torvalds committed
1137
	/*
1138
	 * Filesystem requests must transfer data.
Linus Torvalds's avatar
Linus Torvalds committed
1139
	 */
1140 1141
	BUG_ON(!req->nr_phys_segments);

1142
	memset(cmd->cmnd, 0, BLK_MAX_CDB);
1143
	return scsi_init_io(cmd, GFP_ATOMIC);
1144
}
1145
EXPORT_SYMBOL(scsi_setup_fs_cmnd);
1146

1147 1148
static int
scsi_prep_state_check(struct scsi_device *sdev, struct request *req)
1149 1150 1151
{
	int ret = BLKPREP_OK;

Linus Torvalds's avatar
Linus Torvalds committed
1152
	/*
1153 1154
	 * If the device is not in running state we will reject some
	 * or all commands.
Linus Torvalds's avatar
Linus Torvalds committed
1155
	 */
1156 1157 1158
	if (unlikely(sdev->sdev_state != SDEV_RUNNING)) {
		switch (sdev->sdev_state) {
		case SDEV_OFFLINE:
1159
		case SDEV_TRANSPORT_OFFLINE:
1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173
			/*
			 * If the device is offline we refuse to process any
			 * commands.  The device must be brought online
			 * before trying any recovery commands.
			 */
			sdev_printk(KERN_ERR, sdev,
				    "rejecting I/O to offline device\n");
			ret = BLKPREP_KILL;
			break;
		case SDEV_DEL:
			/*
			 * If the device is fully deleted, we refuse to
			 * process any commands as well.
			 */
1174
			sdev_printk(KERN_ERR, sdev,
1175 1176 1177 1178
				    "rejecting I/O to dead device\n");
			ret = BLKPREP_KILL;
			break;
		case SDEV_BLOCK:
1179
		case SDEV_CREATED_BLOCK:
1180 1181 1182
			ret = BLKPREP_DEFER;
			break;
		case SDEV_QUIESCE:
1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197
			/*
			 * If the devices is blocked we defer normal commands.
			 */
			if (!(req->cmd_flags & REQ_PREEMPT))
				ret = BLKPREP_DEFER;
			break;
		default:
			/*
			 * For any other not fully online state we only allow
			 * special commands.  In particular any user initiated
			 * command is not allowed.
			 */
			if (!(req->cmd_flags & REQ_PREEMPT))
				ret = BLKPREP_KILL;
			break;
Linus Torvalds's avatar
Linus Torvalds committed
1198 1199
		}
	}
1200 1201
	return ret;
}
Linus Torvalds's avatar
Linus Torvalds committed
1202

1203 1204
static int
scsi_prep_return(struct request_queue *q, struct request *req, int ret)
1205 1206
{
	struct scsi_device *sdev = q->queuedata;
Linus Torvalds's avatar
Linus Torvalds committed
1207

1208 1209 1210
	switch (ret) {
	case BLKPREP_KILL:
		req->errors = DID_NO_CONNECT << 16;
1211 1212 1213 1214 1215
		/* release the command and kill it */
		if (req->special) {
			struct scsi_cmnd *cmd = req->special;
			scsi_release_buffers(cmd);
			scsi_put_command(cmd);
1216
			put_device(&sdev->sdev_gendev);
1217 1218
			req->special = NULL;
		}
1219 1220
		break;
	case BLKPREP_DEFER:
Linus Torvalds's avatar
Linus Torvalds committed
1221
		/*
1222
		 * If we defer, the blk_peek_request() returns NULL, but the
1223 1224
		 * queue must be restarted, so we schedule a callback to happen
		 * shortly.
Linus Torvalds's avatar
Linus Torvalds committed
1225
		 */
1226
		if (sdev->device_busy == 0)
1227
			blk_delay_queue(q, SCSI_QUEUE_DELAY);
1228 1229 1230
		break;
	default:
		req->cmd_flags |= REQ_DONTPREP;
Linus Torvalds's avatar
Linus Torvalds committed
1231 1232
	}

1233
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
1234
}
1235

1236
static int scsi_prep_fn(struct request_queue *q, struct request *req)
1237 1238
{
	struct scsi_device *sdev = q->queuedata;
1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250
	struct scsi_cmnd *cmd;
	int ret;

	ret = scsi_prep_state_check(sdev, req);
	if (ret != BLKPREP_OK)
		goto out;

	cmd = scsi_get_cmd_from_req(sdev, req);
	if (unlikely(!cmd)) {
		ret = BLKPREP_DEFER;
		goto out;
	}
1251

1252 1253 1254
	if (req->cmd_type == REQ_TYPE_FS)
		ret = scsi_cmd_to_driver(cmd)->init_command(cmd);
	else if (req->cmd_type == REQ_TYPE_BLOCK_PC)
1255
		ret = scsi_setup_blk_pc_cmnd(sdev, req);
1256 1257 1258 1259
	else
		ret = BLKPREP_KILL;

out:
1260 1261
	return scsi_prep_return(q, req, ret);
}
1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272

static void scsi_unprep_fn(struct request_queue *q, struct request *req)
{
	if (req->cmd_type == REQ_TYPE_FS) {
		struct scsi_cmnd *cmd = req->special;
		struct scsi_driver *drv = scsi_cmd_to_driver(cmd);

		if (drv->uninit_command)
			drv->uninit_command(cmd);
	}
}
Linus Torvalds's avatar
Linus Torvalds committed
1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288

/*
 * scsi_dev_queue_ready: if we can send requests to sdev, return 1 else
 * return 0.
 *
 * Called with the queue_lock held.
 */
static inline int scsi_dev_queue_ready(struct request_queue *q,
				  struct scsi_device *sdev)
{
	if (sdev->device_busy == 0 && sdev->device_blocked) {
		/*
		 * unblock after device_blocked iterates to zero
		 */
		if (--sdev->device_blocked == 0) {
			SCSI_LOG_MLQUEUE(3,
1289 1290
				   sdev_printk(KERN_INFO, sdev,
				   "unblocking device at zero depth\n"));
Linus Torvalds's avatar
Linus Torvalds committed
1291
		} else {
1292
			blk_delay_queue(q, SCSI_QUEUE_DELAY);
Linus Torvalds's avatar
Linus Torvalds committed
1293 1294 1295
			return 0;
		}
	}
1296
	if (scsi_device_is_busy(sdev))
Linus Torvalds's avatar
Linus Torvalds committed
1297 1298 1299 1300 1301
		return 0;

	return 1;
}

1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327

/*
 * scsi_target_queue_ready: checks if there we can send commands to target
 * @sdev: scsi device on starget to check.
 *
 * Called with the host lock held.
 */
static inline int scsi_target_queue_ready(struct Scsi_Host *shost,
					   struct scsi_device *sdev)
{
	struct scsi_target *starget = scsi_target(sdev);

	if (starget->single_lun) {
		if (starget->starget_sdev_user &&
		    starget->starget_sdev_user != sdev)
			return 0;
		starget->starget_sdev_user = sdev;
	}

	if (starget->target_busy == 0 && starget->target_blocked) {
		/*
		 * unblock after target_blocked iterates to zero
		 */
		if (--starget->target_blocked == 0) {
			SCSI_LOG_MLQUEUE(3, starget_printk(KERN_INFO, starget,
					 "unblocking target at zero depth\n"));
1328
		} else
1329 1330 1331 1332
			return 0;
	}

	if (scsi_target_is_busy(starget)) {
1333
		list_move_tail(&sdev->starved_entry, &shost->starved_list);
1334
		return 0;
1335 1336 1337 1338 1339
	}

	return 1;
}

Linus Torvalds's avatar
Linus Torvalds committed
1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350
/*
 * scsi_host_queue_ready: if we can send requests to shost, return 1 else
 * return 0. We must end up running the queue again whenever 0 is
 * returned, else IO can hang.
 *
 * Called with host_lock held.
 */
static inline int scsi_host_queue_ready(struct request_queue *q,
				   struct Scsi_Host *shost,
				   struct scsi_device *sdev)
{
1351
	if (scsi_host_in_recovery(shost))
Linus Torvalds's avatar
Linus Torvalds committed
1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364
		return 0;
	if (shost->host_busy == 0 && shost->host_blocked) {
		/*
		 * unblock after host_blocked iterates to zero
		 */
		if (--shost->host_blocked == 0) {
			SCSI_LOG_MLQUEUE(3,
				printk("scsi%d unblocking host at zero depth\n",
					shost->host_no));
		} else {
			return 0;
		}
	}
1365
	if (scsi_host_is_busy(shost)) {
Linus Torvalds's avatar
Linus Torvalds committed
1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377
		if (list_empty(&sdev->starved_entry))
			list_add_tail(&sdev->starved_entry, &shost->starved_list);
		return 0;
	}

	/* We're OK to process the command, so we can't be starved */
	if (!list_empty(&sdev->starved_entry))
		list_del_init(&sdev->starved_entry);

	return 1;
}

1378 1379 1380 1381 1382 1383 1384 1385
/*
 * Busy state exporting function for request stacking drivers.
 *
 * For efficiency, no lock is taken to check the busy state of
 * shost/starget/sdev, since the returned value is not guaranteed and
 * may be changed after request stacking drivers call the function,
 * regardless of taking lock or not.
 *
1386 1387 1388
 * When scsi can't dispatch I/Os anymore and needs to kill I/Os scsi
 * needs to return 'not busy'. Otherwise, request stacking drivers
 * may hold requests forever.
1389 1390 1391 1392 1393 1394
 */
static int scsi_lld_busy(struct request_queue *q)
{
	struct scsi_device *sdev = q->queuedata;
	struct Scsi_Host *shost;

1395
	if (blk_queue_dying(q))
1396 1397 1398 1399
		return 0;

	shost = sdev->host;

1400 1401 1402 1403 1404 1405 1406
	/*
	 * Ignore host/starget busy state.
	 * Since block layer does not have a concept of fairness across
	 * multiple queues, congestion of host/starget needs to be handled
	 * in SCSI layer.
	 */
	if (scsi_host_in_recovery(shost) || scsi_device_is_busy(sdev))
1407 1408 1409 1410 1411
		return 1;

	return 0;
}

Linus Torvalds's avatar
Linus Torvalds committed
1412
/*
1413
 * Kill a request for a dead device
Linus Torvalds's avatar
Linus Torvalds committed
1414
 */
1415
static void scsi_kill_request(struct request *req, struct request_queue *q)
Linus Torvalds's avatar
Linus Torvalds committed
1416
{
1417
	struct scsi_cmnd *cmd = req->special;
1418 1419 1420
	struct scsi_device *sdev;
	struct scsi_target *starget;
	struct Scsi_Host *shost;
Linus Torvalds's avatar
Linus Torvalds committed
1421

1422
	blk_start_request(req);
1423

1424 1425
	scmd_printk(KERN_INFO, cmd, "killing request\n");

1426 1427 1428
	sdev = cmd->device;
	starget = scsi_target(sdev);
	shost = sdev->host;
1429 1430 1431
	scsi_init_cmd_errh(cmd);
	cmd->result = DID_NO_CONNECT << 16;
	atomic_inc(&cmd->device->iorequest_cnt);
1432 1433 1434 1435 1436 1437 1438 1439 1440 1441

	/*
	 * SCSI request completion path will do scsi_device_unbusy(),
	 * bump busy counts.  To bump the counters, we need to dance
	 * with the locks as normal issue path does.
	 */
	sdev->device_busy++;
	spin_unlock(sdev->request_queue->queue_lock);
	spin_lock(shost->host_lock);
	shost->host_busy++;
1442
	starget->target_busy++;
1443 1444 1445
	spin_unlock(shost->host_lock);
	spin_lock(sdev->request_queue->queue_lock);

1446
	blk_complete_request(req);
Linus Torvalds's avatar
Linus Torvalds committed
1447 1448
}

1449 1450
static void scsi_softirq_done(struct request *rq)
{
1451 1452
	struct scsi_cmnd *cmd = rq->special;
	unsigned long wait_for = (cmd->allowed + 1) * rq->timeout;
1453 1454 1455 1456
	int disposition;

	INIT_LIST_HEAD(&cmd->eh_entry);

1457 1458 1459 1460
	atomic_inc(&cmd->device->iodone_cnt);
	if (cmd->result)
		atomic_inc(&cmd->device->ioerr_cnt);

1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476
	disposition = scsi_decide_disposition(cmd);
	if (disposition != SUCCESS &&
	    time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) {
		sdev_printk(KERN_ERR, cmd->device,
			    "timing out command, waited %lus\n",
			    wait_for/HZ);
		disposition = SUCCESS;
	}
			
	scsi_log_completion(cmd, disposition);

	switch (disposition) {
		case SUCCESS:
			scsi_finish_command(cmd);
			break;
		case NEEDS_RETRY:
1477
			scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY);
1478 1479 1480 1481 1482 1483 1484 1485 1486 1487
			break;
		case ADD_TO_MLQUEUE:
			scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY);
			break;
		default:
			if (!scsi_eh_scmd_add(cmd, 0))
				scsi_finish_command(cmd);
	}
}

Linus Torvalds's avatar
Linus Torvalds committed
1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499
/*
 * Function:    scsi_request_fn()
 *
 * Purpose:     Main strategy routine for SCSI.
 *
 * Arguments:   q       - Pointer to actual queue.
 *
 * Returns:     Nothing
 *
 * Lock status: IO request lock assumed to be held when called.
 */
static void scsi_request_fn(struct request_queue *q)
1500 1501
	__releases(q->queue_lock)
	__acquires(q->queue_lock)
Linus Torvalds's avatar
Linus Torvalds committed
1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512
{
	struct scsi_device *sdev = q->queuedata;
	struct Scsi_Host *shost;
	struct scsi_cmnd *cmd;
	struct request *req;

	/*
	 * To start with, we keep looping until the queue is empty, or until
	 * the host is no longer able to accept any more requests.
	 */
	shost = sdev->host;
1513
	for (;;) {
Linus Torvalds's avatar
Linus Torvalds committed
1514 1515 1516 1517 1518 1519
		int rtn;
		/*
		 * get next queueable request.  We do this early to make sure
		 * that the request is fully prepared even if we cannot 
		 * accept it.
		 */
1520
		req = blk_peek_request(q);
Linus Torvalds's avatar
Linus Torvalds committed
1521 1522 1523 1524
		if (!req || !scsi_dev_queue_ready(q, sdev))
			break;

		if (unlikely(!scsi_device_online(sdev))) {
1525 1526
			sdev_printk(KERN_ERR, sdev,
				    "rejecting I/O to offline device\n");
1527
			scsi_kill_request(req, q);
Linus Torvalds's avatar
Linus Torvalds committed
1528 1529 1530 1531 1532 1533 1534 1535
			continue;
		}


		/*
		 * Remove the request from the request list.
		 */
		if (!(blk_queue_tagged(q) && !blk_queue_start_tag(q, req)))
1536
			blk_start_request(req);
Linus Torvalds's avatar
Linus Torvalds committed
1537 1538 1539
		sdev->device_busy++;

		spin_unlock(q->queue_lock);
1540 1541 1542 1543
		cmd = req->special;
		if (unlikely(cmd == NULL)) {
			printk(KERN_CRIT "impossible request in %s.\n"
					 "please mail a stack trace to "
1544
					 "linux-scsi@vger.kernel.org\n",
1545
					 __func__);
1546
			blk_dump_rq_flags(req, "foo");
1547 1548
			BUG();
		}
Linus Torvalds's avatar
Linus Torvalds committed
1549 1550
		spin_lock(shost->host_lock);

1551 1552 1553 1554 1555 1556 1557 1558
		/*
		 * We hit this when the driver is using a host wide
		 * tag map. For device level tag maps the queue_depth check
		 * in the device ready fn would prevent us from trying
		 * to allocate a tag. Since the map is a shared host resource
		 * we add the dev to the starved list so it eventually gets
		 * a run when a tag is freed.
		 */
1559
		if (blk_queue_tagged(q) && !blk_rq_tagged(req)) {
1560 1561 1562 1563 1564 1565
			if (list_empty(&sdev->starved_entry))
				list_add_tail(&sdev->starved_entry,
					      &shost->starved_list);
			goto not_ready;
		}

1566 1567 1568
		if (!scsi_target_queue_ready(shost, sdev))
			goto not_ready;

Linus Torvalds's avatar
Linus Torvalds committed
1569 1570
		if (!scsi_host_queue_ready(q, shost, sdev))
			goto not_ready;
1571 1572

		scsi_target(sdev)->target_busy++;
Linus Torvalds's avatar
Linus Torvalds committed
1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591
		shost->host_busy++;

		/*
		 * XXX(hch): This is rather suboptimal, scsi_dispatch_cmd will
		 *		take the lock again.
		 */
		spin_unlock_irq(shost->host_lock);

		/*
		 * Finally, initialize any error handling parameters, and set up
		 * the timers for timeouts.
		 */
		scsi_init_cmd_errh(cmd);

		/*
		 * Dispatch the command to the low-level driver.
		 */
		rtn = scsi_dispatch_cmd(cmd);
		spin_lock_irq(q->queue_lock);
1592 1593
		if (rtn)
			goto out_delay;
Linus Torvalds's avatar
Linus Torvalds committed
1594 1595
	}

1596
	return;
Linus Torvalds's avatar
Linus Torvalds committed
1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611

 not_ready:
	spin_unlock_irq(shost->host_lock);

	/*
	 * lock q, handle tag, requeue req, and decrement device_busy. We
	 * must return with queue_lock held.
	 *
	 * Decrementing device_busy without checking it is OK, as all such
	 * cases (host limits or settings) should run the queue at some
	 * later time.
	 */
	spin_lock_irq(q->queue_lock);
	blk_requeue_request(q, req);
	sdev->device_busy--;
1612 1613 1614
out_delay:
	if (sdev->device_busy == 0)
		blk_delay_queue(q, SCSI_QUEUE_DELAY);
Linus Torvalds's avatar
Linus Torvalds committed
1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632
}

u64 scsi_calculate_bounce_limit(struct Scsi_Host *shost)
{
	struct device *host_dev;
	u64 bounce_limit = 0xffffffff;

	if (shost->unchecked_isa_dma)
		return BLK_BOUNCE_ISA;
	/*
	 * Platforms with virtual-DMA translation
	 * hardware have no practical limit.
	 */
	if (!PCI_DMA_BUS_IS_PHYS)
		return BLK_BOUNCE_ANY;

	host_dev = scsi_get_device(shost);
	if (host_dev && host_dev->dma_mask)
1633
		bounce_limit = (u64)dma_max_pfn(host_dev) << PAGE_SHIFT;
Linus Torvalds's avatar
Linus Torvalds committed
1634 1635 1636 1637 1638

	return bounce_limit;
}
EXPORT_SYMBOL(scsi_calculate_bounce_limit);

1639 1640
struct request_queue *__scsi_alloc_queue(struct Scsi_Host *shost,
					 request_fn_proc *request_fn)
Linus Torvalds's avatar
Linus Torvalds committed
1641 1642
{
	struct request_queue *q;
1643
	struct device *dev = shost->dma_dev;
Linus Torvalds's avatar
Linus Torvalds committed
1644

1645
	q = blk_init_queue(request_fn, NULL);
Linus Torvalds's avatar
Linus Torvalds committed
1646 1647 1648
	if (!q)
		return NULL;

1649 1650 1651
	/*
	 * this limit is imposed by hardware restrictions
	 */
1652 1653
	blk_queue_max_segments(q, min_t(unsigned short, shost->sg_tablesize,
					SCSI_MAX_SG_CHAIN_SEGMENTS));
1654

1655 1656 1657 1658 1659 1660 1661 1662
	if (scsi_host_prot_dma(shost)) {
		shost->sg_prot_tablesize =
			min_not_zero(shost->sg_prot_tablesize,
				     (unsigned short)SCSI_MAX_PROT_SG_SEGMENTS);
		BUG_ON(shost->sg_prot_tablesize < shost->sg_tablesize);
		blk_queue_max_integrity_segments(q, shost->sg_prot_tablesize);
	}

1663
	blk_queue_max_hw_sectors(q, shost->max_sectors);
Linus Torvalds's avatar
Linus Torvalds committed
1664 1665
	blk_queue_bounce_limit(q, scsi_calculate_bounce_limit(shost));
	blk_queue_segment_boundary(q, shost->dma_boundary);
1666
	dma_set_seg_boundary(dev, shost->dma_boundary);
Linus Torvalds's avatar
Linus Torvalds committed
1667

1668 1669
	blk_queue_max_segment_size(q, dma_get_max_seg_size(dev));

Linus Torvalds's avatar
Linus Torvalds committed
1670
	if (!shost->use_clustering)
1671
		q->limits.cluster = 0;
1672 1673 1674 1675 1676 1677 1678 1679

	/*
	 * set a reasonable default alignment on word boundaries: the
	 * host and device may alter it using
	 * blk_queue_update_dma_alignment() later.
	 */
	blk_queue_dma_alignment(q, 0x03);

Linus Torvalds's avatar
Linus Torvalds committed
1680 1681
	return q;
}
1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692
EXPORT_SYMBOL(__scsi_alloc_queue);

struct request_queue *scsi_alloc_queue(struct scsi_device *sdev)
{
	struct request_queue *q;

	q = __scsi_alloc_queue(sdev->host, scsi_request_fn);
	if (!q)
		return NULL;

	blk_queue_prep_rq(q, scsi_prep_fn);
1693
	blk_queue_unprep_rq(q, scsi_unprep_fn);
1694
	blk_queue_softirq_done(q, scsi_softirq_done);
1695
	blk_queue_rq_timed_out(q, scsi_times_out);
1696
	blk_queue_lld_busy(q, scsi_lld_busy);
1697 1698
	return q;
}
Linus Torvalds's avatar
Linus Torvalds committed
1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752

/*
 * Function:    scsi_block_requests()
 *
 * Purpose:     Utility function used by low-level drivers to prevent further
 *		commands from being queued to the device.
 *
 * Arguments:   shost       - Host in question
 *
 * Returns:     Nothing
 *
 * Lock status: No locks are assumed held.
 *
 * Notes:       There is no timer nor any other means by which the requests
 *		get unblocked other than the low-level driver calling
 *		scsi_unblock_requests().
 */
void scsi_block_requests(struct Scsi_Host *shost)
{
	shost->host_self_blocked = 1;
}
EXPORT_SYMBOL(scsi_block_requests);

/*
 * Function:    scsi_unblock_requests()
 *
 * Purpose:     Utility function used by low-level drivers to allow further
 *		commands from being queued to the device.
 *
 * Arguments:   shost       - Host in question
 *
 * Returns:     Nothing
 *
 * Lock status: No locks are assumed held.
 *
 * Notes:       There is no timer nor any other means by which the requests
 *		get unblocked other than the low-level driver calling
 *		scsi_unblock_requests().
 *
 *		This is done as an API function so that changes to the
 *		internals of the scsi mid-layer won't require wholesale
 *		changes to drivers that use this feature.
 */
void scsi_unblock_requests(struct Scsi_Host *shost)
{
	shost->host_self_blocked = 0;
	scsi_run_host_queues(shost);
}
EXPORT_SYMBOL(scsi_unblock_requests);

int __init scsi_init_queue(void)
{
	int i;

1753 1754 1755 1756 1757
	scsi_sdb_cache = kmem_cache_create("scsi_data_buffer",
					   sizeof(struct scsi_data_buffer),
					   0, 0, NULL);
	if (!scsi_sdb_cache) {
		printk(KERN_ERR "SCSI: can't init scsi sdb cache\n");
1758
		return -ENOMEM;
1759 1760
	}

Linus Torvalds's avatar
Linus Torvalds committed
1761 1762 1763 1764 1765
	for (i = 0; i < SG_MEMPOOL_NR; i++) {
		struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
		int size = sgp->size * sizeof(struct scatterlist);

		sgp->slab = kmem_cache_create(sgp->name, size, 0,
1766
				SLAB_HWCACHE_ALIGN, NULL);
Linus Torvalds's avatar
Linus Torvalds committed
1767 1768 1769
		if (!sgp->slab) {
			printk(KERN_ERR "SCSI: can't init sg slab %s\n",
					sgp->name);
1770
			goto cleanup_sdb;
Linus Torvalds's avatar
Linus Torvalds committed
1771 1772
		}

1773 1774
		sgp->pool = mempool_create_slab_pool(SG_MEMPOOL_SIZE,
						     sgp->slab);
Linus Torvalds's avatar
Linus Torvalds committed
1775 1776 1777
		if (!sgp->pool) {
			printk(KERN_ERR "SCSI: can't init sg mempool %s\n",
					sgp->name);
1778
			goto cleanup_sdb;
Linus Torvalds's avatar
Linus Torvalds committed
1779 1780 1781 1782
		}
	}

	return 0;
1783

1784
cleanup_sdb:
1785 1786 1787 1788 1789 1790 1791
	for (i = 0; i < SG_MEMPOOL_NR; i++) {
		struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
		if (sgp->pool)
			mempool_destroy(sgp->pool);
		if (sgp->slab)
			kmem_cache_destroy(sgp->slab);
	}
1792
	kmem_cache_destroy(scsi_sdb_cache);
1793 1794

	return -ENOMEM;
Linus Torvalds's avatar
Linus Torvalds committed
1795 1796 1797 1798 1799 1800
}

void scsi_exit_queue(void)
{
	int i;

1801
	kmem_cache_destroy(scsi_sdb_cache);
1802

Linus Torvalds's avatar
Linus Torvalds committed
1803 1804 1805 1806 1807 1808
	for (i = 0; i < SG_MEMPOOL_NR; i++) {
		struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
		mempool_destroy(sgp->pool);
		kmem_cache_destroy(sgp->slab);
	}
}
1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820

/**
 *	scsi_mode_select - issue a mode select
 *	@sdev:	SCSI device to be queried
 *	@pf:	Page format bit (1 == standard, 0 == vendor specific)
 *	@sp:	Save page bit (0 == don't save, 1 == save)
 *	@modepage: mode page being requested
 *	@buffer: request buffer (may not be smaller than eight bytes)
 *	@len:	length of request buffer.
 *	@timeout: command timeout
 *	@retries: number of retries before failing
 *	@data: returns a structure abstracting the mode header data
1821
 *	@sshdr: place to put sense data (or NULL if no sense to be collected).
1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880
 *		must be SCSI_SENSE_BUFFERSIZE big.
 *
 *	Returns zero if successful; negative error number or scsi
 *	status on error
 *
 */
int
scsi_mode_select(struct scsi_device *sdev, int pf, int sp, int modepage,
		 unsigned char *buffer, int len, int timeout, int retries,
		 struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr)
{
	unsigned char cmd[10];
	unsigned char *real_buffer;
	int ret;

	memset(cmd, 0, sizeof(cmd));
	cmd[1] = (pf ? 0x10 : 0) | (sp ? 0x01 : 0);

	if (sdev->use_10_for_ms) {
		if (len > 65535)
			return -EINVAL;
		real_buffer = kmalloc(8 + len, GFP_KERNEL);
		if (!real_buffer)
			return -ENOMEM;
		memcpy(real_buffer + 8, buffer, len);
		len += 8;
		real_buffer[0] = 0;
		real_buffer[1] = 0;
		real_buffer[2] = data->medium_type;
		real_buffer[3] = data->device_specific;
		real_buffer[4] = data->longlba ? 0x01 : 0;
		real_buffer[5] = 0;
		real_buffer[6] = data->block_descriptor_length >> 8;
		real_buffer[7] = data->block_descriptor_length;

		cmd[0] = MODE_SELECT_10;
		cmd[7] = len >> 8;
		cmd[8] = len;
	} else {
		if (len > 255 || data->block_descriptor_length > 255 ||
		    data->longlba)
			return -EINVAL;

		real_buffer = kmalloc(4 + len, GFP_KERNEL);
		if (!real_buffer)
			return -ENOMEM;
		memcpy(real_buffer + 4, buffer, len);
		len += 4;
		real_buffer[0] = 0;
		real_buffer[1] = data->medium_type;
		real_buffer[2] = data->device_specific;
		real_buffer[3] = data->block_descriptor_length;
		

		cmd[0] = MODE_SELECT;
		cmd[4] = len;
	}

	ret = scsi_execute_req(sdev, cmd, DMA_TO_DEVICE, real_buffer, len,
1881
			       sshdr, timeout, retries, NULL);
1882 1883 1884 1885 1886
	kfree(real_buffer);
	return ret;
}
EXPORT_SYMBOL_GPL(scsi_mode_select);

Linus Torvalds's avatar
Linus Torvalds committed
1887
/**
1888
 *	scsi_mode_sense - issue a mode sense, falling back from 10 to six bytes if necessary.
1889
 *	@sdev:	SCSI device to be queried
Linus Torvalds's avatar
Linus Torvalds committed
1890 1891 1892 1893 1894 1895 1896
 *	@dbd:	set if mode sense will allow block descriptors to be returned
 *	@modepage: mode page being requested
 *	@buffer: request buffer (may not be smaller than eight bytes)
 *	@len:	length of request buffer.
 *	@timeout: command timeout
 *	@retries: number of retries before failing
 *	@data: returns a structure abstracting the mode header data
1897
 *	@sshdr: place to put sense data (or NULL if no sense to be collected).
1898
 *		must be SCSI_SENSE_BUFFERSIZE big.
Linus Torvalds's avatar
Linus Torvalds committed
1899 1900 1901 1902
 *
 *	Returns zero if unsuccessful, or the header offset (either 4
 *	or 8 depending on whether a six or ten byte command was
 *	issued) if successful.
1903
 */
Linus Torvalds's avatar
Linus Torvalds committed
1904
int
1905
scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage,
Linus Torvalds's avatar
Linus Torvalds committed
1906
		  unsigned char *buffer, int len, int timeout, int retries,
1907 1908
		  struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr)
{
Linus Torvalds's avatar
Linus Torvalds committed
1909 1910 1911
	unsigned char cmd[12];
	int use_10_for_ms;
	int header_length;
1912
	int result;
1913
	struct scsi_sense_hdr my_sshdr;
Linus Torvalds's avatar
Linus Torvalds committed
1914 1915 1916 1917 1918 1919

	memset(data, 0, sizeof(*data));
	memset(&cmd[0], 0, 12);
	cmd[1] = dbd & 0x18;	/* allows DBD and LLBA bits */
	cmd[2] = modepage;

1920 1921 1922 1923
	/* caller might not be interested in sense, but we need it */
	if (!sshdr)
		sshdr = &my_sshdr;

Linus Torvalds's avatar
Linus Torvalds committed
1924
 retry:
1925
	use_10_for_ms = sdev->use_10_for_ms;
Linus Torvalds's avatar
Linus Torvalds committed
1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944

	if (use_10_for_ms) {
		if (len < 8)
			len = 8;

		cmd[0] = MODE_SENSE_10;
		cmd[8] = len;
		header_length = 8;
	} else {
		if (len < 4)
			len = 4;

		cmd[0] = MODE_SENSE;
		cmd[4] = len;
		header_length = 4;
	}

	memset(buffer, 0, len);

1945
	result = scsi_execute_req(sdev, cmd, DMA_FROM_DEVICE, buffer, len,
1946
				  sshdr, timeout, retries, NULL);
Linus Torvalds's avatar
Linus Torvalds committed
1947 1948 1949 1950 1951 1952

	/* This code looks awful: what it's doing is making sure an
	 * ILLEGAL REQUEST sense return identifies the actual command
	 * byte as the problem.  MODE_SENSE commands can return
	 * ILLEGAL REQUEST if the code page isn't supported */

1953 1954
	if (use_10_for_ms && !scsi_status_is_good(result) &&
	    (driver_byte(result) & DRIVER_SENSE)) {
1955 1956 1957
		if (scsi_sense_valid(sshdr)) {
			if ((sshdr->sense_key == ILLEGAL_REQUEST) &&
			    (sshdr->asc == 0x20) && (sshdr->ascq == 0)) {
Linus Torvalds's avatar
Linus Torvalds committed
1958 1959 1960
				/* 
				 * Invalid command operation code
				 */
1961
				sdev->use_10_for_ms = 0;
Linus Torvalds's avatar
Linus Torvalds committed
1962 1963 1964 1965 1966
				goto retry;
			}
		}
	}

1967
	if(scsi_status_is_good(result)) {
1968 1969 1970 1971 1972 1973 1974 1975 1976 1977
		if (unlikely(buffer[0] == 0x86 && buffer[1] == 0x0b &&
			     (modepage == 6 || modepage == 8))) {
			/* Initio breakage? */
			header_length = 0;
			data->length = 13;
			data->medium_type = 0;
			data->device_specific = 0;
			data->longlba = 0;
			data->block_descriptor_length = 0;
		} else if(use_10_for_ms) {
Linus Torvalds's avatar
Linus Torvalds committed
1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989
			data->length = buffer[0]*256 + buffer[1] + 2;
			data->medium_type = buffer[2];
			data->device_specific = buffer[3];
			data->longlba = buffer[4] & 0x01;
			data->block_descriptor_length = buffer[6]*256
				+ buffer[7];
		} else {
			data->length = buffer[0] + 1;
			data->medium_type = buffer[1];
			data->device_specific = buffer[2];
			data->block_descriptor_length = buffer[3];
		}
1990
		data->header_length = header_length;
Linus Torvalds's avatar
Linus Torvalds committed
1991 1992
	}

1993
	return result;
Linus Torvalds's avatar
Linus Torvalds committed
1994 1995 1996
}
EXPORT_SYMBOL(scsi_mode_sense);

1997 1998 1999 2000 2001 2002 2003 2004 2005 2006
/**
 *	scsi_test_unit_ready - test if unit is ready
 *	@sdev:	scsi device to change the state of.
 *	@timeout: command timeout
 *	@retries: number of retries before failing
 *	@sshdr_external: Optional pointer to struct scsi_sense_hdr for
 *		returning sense. Make sure that this is cleared before passing
 *		in.
 *
 *	Returns zero if unsuccessful or an error if TUR failed.  For
2007
 *	removable media, UNIT_ATTENTION sets ->changed flag.
2008
 **/
Linus Torvalds's avatar
Linus Torvalds committed
2009
int
2010 2011
scsi_test_unit_ready(struct scsi_device *sdev, int timeout, int retries,
		     struct scsi_sense_hdr *sshdr_external)
Linus Torvalds's avatar
Linus Torvalds committed
2012 2013 2014 2015
{
	char cmd[] = {
		TEST_UNIT_READY, 0, 0, 0, 0, 0,
	};
2016
	struct scsi_sense_hdr *sshdr;
Linus Torvalds's avatar
Linus Torvalds committed
2017
	int result;
2018 2019 2020 2021 2022 2023 2024 2025 2026

	if (!sshdr_external)
		sshdr = kzalloc(sizeof(*sshdr), GFP_KERNEL);
	else
		sshdr = sshdr_external;

	/* try to eat the UNIT_ATTENTION if there are enough retries */
	do {
		result = scsi_execute_req(sdev, cmd, DMA_NONE, NULL, 0, sshdr,
2027
					  timeout, retries, NULL);
2028 2029 2030 2031 2032
		if (sdev->removable && scsi_sense_valid(sshdr) &&
		    sshdr->sense_key == UNIT_ATTENTION)
			sdev->changed = 1;
	} while (scsi_sense_valid(sshdr) &&
		 sshdr->sense_key == UNIT_ATTENTION && --retries);
2033 2034 2035

	if (!sshdr_external)
		kfree(sshdr);
Linus Torvalds's avatar
Linus Torvalds committed
2036 2037 2038 2039 2040
	return result;
}
EXPORT_SYMBOL(scsi_test_unit_ready);

/**
2041
 *	scsi_device_set_state - Take the given device through the device state model.
Linus Torvalds's avatar
Linus Torvalds committed
2042 2043 2044 2045 2046
 *	@sdev:	scsi device to change the state of.
 *	@state:	state to change to.
 *
 *	Returns zero if unsuccessful or an error if the requested 
 *	transition is illegal.
2047
 */
Linus Torvalds's avatar
Linus Torvalds committed
2048 2049 2050 2051 2052 2053 2054 2055 2056 2057
int
scsi_device_set_state(struct scsi_device *sdev, enum scsi_device_state state)
{
	enum scsi_device_state oldstate = sdev->sdev_state;

	if (state == oldstate)
		return 0;

	switch (state) {
	case SDEV_CREATED:
2058 2059 2060 2061 2062 2063 2064
		switch (oldstate) {
		case SDEV_CREATED_BLOCK:
			break;
		default:
			goto illegal;
		}
		break;
Linus Torvalds's avatar
Linus Torvalds committed
2065 2066 2067 2068 2069
			
	case SDEV_RUNNING:
		switch (oldstate) {
		case SDEV_CREATED:
		case SDEV_OFFLINE:
2070
		case SDEV_TRANSPORT_OFFLINE:
Linus Torvalds's avatar
Linus Torvalds committed
2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082
		case SDEV_QUIESCE:
		case SDEV_BLOCK:
			break;
		default:
			goto illegal;
		}
		break;

	case SDEV_QUIESCE:
		switch (oldstate) {
		case SDEV_RUNNING:
		case SDEV_OFFLINE:
2083
		case SDEV_TRANSPORT_OFFLINE:
Linus Torvalds's avatar
Linus Torvalds committed
2084 2085 2086 2087 2088 2089 2090
			break;
		default:
			goto illegal;
		}
		break;

	case SDEV_OFFLINE:
2091
	case SDEV_TRANSPORT_OFFLINE:
Linus Torvalds's avatar
Linus Torvalds committed
2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105
		switch (oldstate) {
		case SDEV_CREATED:
		case SDEV_RUNNING:
		case SDEV_QUIESCE:
		case SDEV_BLOCK:
			break;
		default:
			goto illegal;
		}
		break;

	case SDEV_BLOCK:
		switch (oldstate) {
		case SDEV_RUNNING:
2106 2107 2108 2109 2110 2111 2112 2113 2114 2115
		case SDEV_CREATED_BLOCK:
			break;
		default:
			goto illegal;
		}
		break;

	case SDEV_CREATED_BLOCK:
		switch (oldstate) {
		case SDEV_CREATED:
Linus Torvalds's avatar
Linus Torvalds committed
2116 2117 2118 2119 2120 2121 2122 2123 2124 2125
			break;
		default:
			goto illegal;
		}
		break;

	case SDEV_CANCEL:
		switch (oldstate) {
		case SDEV_CREATED:
		case SDEV_RUNNING:
2126
		case SDEV_QUIESCE:
Linus Torvalds's avatar
Linus Torvalds committed
2127
		case SDEV_OFFLINE:
2128
		case SDEV_TRANSPORT_OFFLINE:
Linus Torvalds's avatar
Linus Torvalds committed
2129 2130 2131 2132 2133 2134 2135 2136 2137
		case SDEV_BLOCK:
			break;
		default:
			goto illegal;
		}
		break;

	case SDEV_DEL:
		switch (oldstate) {
2138 2139 2140
		case SDEV_CREATED:
		case SDEV_RUNNING:
		case SDEV_OFFLINE:
2141
		case SDEV_TRANSPORT_OFFLINE:
Linus Torvalds's avatar
Linus Torvalds committed
2142
		case SDEV_CANCEL:
2143
		case SDEV_CREATED_BLOCK:
Linus Torvalds's avatar
Linus Torvalds committed
2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155
			break;
		default:
			goto illegal;
		}
		break;

	}
	sdev->sdev_state = state;
	return 0;

 illegal:
	SCSI_LOG_ERROR_RECOVERY(1, 
2156 2157 2158 2159
				sdev_printk(KERN_ERR, sdev,
					    "Illegal state transition %s->%s\n",
					    scsi_device_state_name(oldstate),
					    scsi_device_state_name(state))
Linus Torvalds's avatar
Linus Torvalds committed
2160 2161 2162 2163 2164
				);
	return -EINVAL;
}
EXPORT_SYMBOL(scsi_device_set_state);

2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180
/**
 * 	sdev_evt_emit - emit a single SCSI device uevent
 *	@sdev: associated SCSI device
 *	@evt: event to emit
 *
 *	Send a single uevent (scsi_event) to the associated scsi_device.
 */
static void scsi_evt_emit(struct scsi_device *sdev, struct scsi_event *evt)
{
	int idx = 0;
	char *envp[3];

	switch (evt->evt_type) {
	case SDEV_EVT_MEDIA_CHANGE:
		envp[idx++] = "SDEV_MEDIA_CHANGE=1";
		break;
2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195
	case SDEV_EVT_INQUIRY_CHANGE_REPORTED:
		envp[idx++] = "SDEV_UA=INQUIRY_DATA_HAS_CHANGED";
		break;
	case SDEV_EVT_CAPACITY_CHANGE_REPORTED:
		envp[idx++] = "SDEV_UA=CAPACITY_DATA_HAS_CHANGED";
		break;
	case SDEV_EVT_SOFT_THRESHOLD_REACHED_REPORTED:
	       envp[idx++] = "SDEV_UA=THIN_PROVISIONING_SOFT_THRESHOLD_REACHED";
		break;
	case SDEV_EVT_MODE_PARAMETER_CHANGE_REPORTED:
		envp[idx++] = "SDEV_UA=MODE_PARAMETERS_CHANGED";
		break;
	case SDEV_EVT_LUN_CHANGE_REPORTED:
		envp[idx++] = "SDEV_UA=REPORTED_LUNS_DATA_HAS_CHANGED";
		break;
2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215
	default:
		/* do nothing */
		break;
	}

	envp[idx++] = NULL;

	kobject_uevent_env(&sdev->sdev_gendev.kobj, KOBJ_CHANGE, envp);
}

/**
 * 	sdev_evt_thread - send a uevent for each scsi event
 *	@work: work struct for scsi_device
 *
 *	Dispatch queued events to their associated scsi_device kobjects
 *	as uevents.
 */
void scsi_evt_thread(struct work_struct *work)
{
	struct scsi_device *sdev;
2216
	enum scsi_device_event evt_type;
2217 2218 2219 2220
	LIST_HEAD(event_list);

	sdev = container_of(work, struct scsi_device, event_work);

2221 2222 2223 2224
	for (evt_type = SDEV_EVT_FIRST; evt_type <= SDEV_EVT_LAST; evt_type++)
		if (test_and_clear_bit(evt_type, sdev->pending_events))
			sdev_evt_send_simple(sdev, evt_type, GFP_KERNEL);

2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256
	while (1) {
		struct scsi_event *evt;
		struct list_head *this, *tmp;
		unsigned long flags;

		spin_lock_irqsave(&sdev->list_lock, flags);
		list_splice_init(&sdev->event_list, &event_list);
		spin_unlock_irqrestore(&sdev->list_lock, flags);

		if (list_empty(&event_list))
			break;

		list_for_each_safe(this, tmp, &event_list) {
			evt = list_entry(this, struct scsi_event, node);
			list_del(&evt->node);
			scsi_evt_emit(sdev, evt);
			kfree(evt);
		}
	}
}

/**
 * 	sdev_evt_send - send asserted event to uevent thread
 *	@sdev: scsi_device event occurred on
 *	@evt: event to send
 *
 *	Assert scsi device event asynchronously.
 */
void sdev_evt_send(struct scsi_device *sdev, struct scsi_event *evt)
{
	unsigned long flags;

2257 2258 2259 2260
#if 0
	/* FIXME: currently this check eliminates all media change events
	 * for polled devices.  Need to update to discriminate between AN
	 * and polled events */
2261 2262 2263 2264
	if (!test_bit(evt->evt_type, sdev->supported_events)) {
		kfree(evt);
		return;
	}
2265
#endif
2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293

	spin_lock_irqsave(&sdev->list_lock, flags);
	list_add_tail(&evt->node, &sdev->event_list);
	schedule_work(&sdev->event_work);
	spin_unlock_irqrestore(&sdev->list_lock, flags);
}
EXPORT_SYMBOL_GPL(sdev_evt_send);

/**
 * 	sdev_evt_alloc - allocate a new scsi event
 *	@evt_type: type of event to allocate
 *	@gfpflags: GFP flags for allocation
 *
 *	Allocates and returns a new scsi_event.
 */
struct scsi_event *sdev_evt_alloc(enum scsi_device_event evt_type,
				  gfp_t gfpflags)
{
	struct scsi_event *evt = kzalloc(sizeof(struct scsi_event), gfpflags);
	if (!evt)
		return NULL;

	evt->evt_type = evt_type;
	INIT_LIST_HEAD(&evt->node);

	/* evt_type-specific initialization, if any */
	switch (evt_type) {
	case SDEV_EVT_MEDIA_CHANGE:
2294 2295 2296 2297 2298
	case SDEV_EVT_INQUIRY_CHANGE_REPORTED:
	case SDEV_EVT_CAPACITY_CHANGE_REPORTED:
	case SDEV_EVT_SOFT_THRESHOLD_REACHED_REPORTED:
	case SDEV_EVT_MODE_PARAMETER_CHANGE_REPORTED:
	case SDEV_EVT_LUN_CHANGE_REPORTED:
2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329
	default:
		/* do nothing */
		break;
	}

	return evt;
}
EXPORT_SYMBOL_GPL(sdev_evt_alloc);

/**
 * 	sdev_evt_send_simple - send asserted event to uevent thread
 *	@sdev: scsi_device event occurred on
 *	@evt_type: type of event to send
 *	@gfpflags: GFP flags for allocation
 *
 *	Assert scsi device event asynchronously, given an event type.
 */
void sdev_evt_send_simple(struct scsi_device *sdev,
			  enum scsi_device_event evt_type, gfp_t gfpflags)
{
	struct scsi_event *evt = sdev_evt_alloc(evt_type, gfpflags);
	if (!evt) {
		sdev_printk(KERN_ERR, sdev, "event %d eaten due to OOM\n",
			    evt_type);
		return;
	}

	sdev_evt_send(sdev, evt);
}
EXPORT_SYMBOL_GPL(sdev_evt_send_simple);

Linus Torvalds's avatar
Linus Torvalds committed
2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343
/**
 *	scsi_device_quiesce - Block user issued commands.
 *	@sdev:	scsi device to quiesce.
 *
 *	This works by trying to transition to the SDEV_QUIESCE state
 *	(which must be a legal transition).  When the device is in this
 *	state, only special requests will be accepted, all others will
 *	be deferred.  Since special requests may also be requeued requests,
 *	a successful return doesn't guarantee the device will be 
 *	totally quiescent.
 *
 *	Must be called with user context, may sleep.
 *
 *	Returns zero if unsuccessful or an error if not.
2344
 */
Linus Torvalds's avatar
Linus Torvalds committed
2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368
int
scsi_device_quiesce(struct scsi_device *sdev)
{
	int err = scsi_device_set_state(sdev, SDEV_QUIESCE);
	if (err)
		return err;

	scsi_run_queue(sdev->request_queue);
	while (sdev->device_busy) {
		msleep_interruptible(200);
		scsi_run_queue(sdev->request_queue);
	}
	return 0;
}
EXPORT_SYMBOL(scsi_device_quiesce);

/**
 *	scsi_device_resume - Restart user issued commands to a quiesced device.
 *	@sdev:	scsi device to resume.
 *
 *	Moves the device from quiesced back to running and restarts the
 *	queues.
 *
 *	Must be called with user context, may sleep.
2369
 */
2370
void scsi_device_resume(struct scsi_device *sdev)
Linus Torvalds's avatar
Linus Torvalds committed
2371
{
2372 2373 2374 2375 2376 2377
	/* check if the device state was mutated prior to resume, and if
	 * so assume the state is being managed elsewhere (for example
	 * device deleted during suspend)
	 */
	if (sdev->sdev_state != SDEV_QUIESCE ||
	    scsi_device_set_state(sdev, SDEV_RUNNING))
Linus Torvalds's avatar
Linus Torvalds committed
2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409
		return;
	scsi_run_queue(sdev->request_queue);
}
EXPORT_SYMBOL(scsi_device_resume);

static void
device_quiesce_fn(struct scsi_device *sdev, void *data)
{
	scsi_device_quiesce(sdev);
}

void
scsi_target_quiesce(struct scsi_target *starget)
{
	starget_for_each_device(starget, NULL, device_quiesce_fn);
}
EXPORT_SYMBOL(scsi_target_quiesce);

static void
device_resume_fn(struct scsi_device *sdev, void *data)
{
	scsi_device_resume(sdev);
}

void
scsi_target_resume(struct scsi_target *starget)
{
	starget_for_each_device(starget, NULL, device_resume_fn);
}
EXPORT_SYMBOL(scsi_target_resume);

/**
2410
 * scsi_internal_device_block - internal function to put a device temporarily into the SDEV_BLOCK state
Linus Torvalds's avatar
Linus Torvalds committed
2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423
 * @sdev:	device to block
 *
 * Block request made by scsi lld's to temporarily stop all
 * scsi commands on the specified device.  Called from interrupt
 * or normal process context.
 *
 * Returns zero if successful or error if not
 *
 * Notes:       
 *	This routine transitions the device to the SDEV_BLOCK state
 *	(which must be a legal transition).  When the device is in this
 *	state, all commands are deferred until the scsi lld reenables
 *	the device with scsi_device_unblock or device_block_tmo fires.
2424
 */
Linus Torvalds's avatar
Linus Torvalds committed
2425 2426 2427
int
scsi_internal_device_block(struct scsi_device *sdev)
{
2428
	struct request_queue *q = sdev->request_queue;
Linus Torvalds's avatar
Linus Torvalds committed
2429 2430 2431 2432
	unsigned long flags;
	int err = 0;

	err = scsi_device_set_state(sdev, SDEV_BLOCK);
2433 2434 2435 2436 2437 2438
	if (err) {
		err = scsi_device_set_state(sdev, SDEV_CREATED_BLOCK);

		if (err)
			return err;
	}
Linus Torvalds's avatar
Linus Torvalds committed
2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455

	/* 
	 * The device has transitioned to SDEV_BLOCK.  Stop the
	 * block layer from calling the midlayer with this device's
	 * request queue. 
	 */
	spin_lock_irqsave(q->queue_lock, flags);
	blk_stop_queue(q);
	spin_unlock_irqrestore(q->queue_lock, flags);

	return 0;
}
EXPORT_SYMBOL_GPL(scsi_internal_device_block);
 
/**
 * scsi_internal_device_unblock - resume a device after a block request
 * @sdev:	device to resume
2456
 * @new_state:	state to set devices to after unblocking
Linus Torvalds's avatar
Linus Torvalds committed
2457 2458 2459 2460 2461 2462 2463 2464 2465
 *
 * Called by scsi lld's or the midlayer to restart the device queue
 * for the previously suspended scsi device.  Called from interrupt or
 * normal process context.
 *
 * Returns zero if successful or error if not.
 *
 * Notes:       
 *	This routine transitions the device to the SDEV_RUNNING state
2466
 *	or to one of the offline states (which must be a legal transition)
2467
 *	allowing the midlayer to goose the queue for this device.
2468
 */
Linus Torvalds's avatar
Linus Torvalds committed
2469
int
2470 2471
scsi_internal_device_unblock(struct scsi_device *sdev,
			     enum scsi_device_state new_state)
Linus Torvalds's avatar
Linus Torvalds committed
2472
{
2473
	struct request_queue *q = sdev->request_queue; 
Linus Torvalds's avatar
Linus Torvalds committed
2474
	unsigned long flags;
2475 2476 2477 2478

	/*
	 * Try to transition the scsi device to SDEV_RUNNING or one of the
	 * offlined states and goose the device queue if successful.
Linus Torvalds's avatar
Linus Torvalds committed
2479
	 */
2480 2481
	if ((sdev->sdev_state == SDEV_BLOCK) ||
	    (sdev->sdev_state == SDEV_TRANSPORT_OFFLINE))
2482 2483 2484 2485 2486 2487 2488 2489
		sdev->sdev_state = new_state;
	else if (sdev->sdev_state == SDEV_CREATED_BLOCK) {
		if (new_state == SDEV_TRANSPORT_OFFLINE ||
		    new_state == SDEV_OFFLINE)
			sdev->sdev_state = new_state;
		else
			sdev->sdev_state = SDEV_CREATED;
	} else if (sdev->sdev_state != SDEV_CANCEL &&
2490
		 sdev->sdev_state != SDEV_OFFLINE)
2491
		return -EINVAL;
Linus Torvalds's avatar
Linus Torvalds committed
2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529

	spin_lock_irqsave(q->queue_lock, flags);
	blk_start_queue(q);
	spin_unlock_irqrestore(q->queue_lock, flags);

	return 0;
}
EXPORT_SYMBOL_GPL(scsi_internal_device_unblock);

static void
device_block(struct scsi_device *sdev, void *data)
{
	scsi_internal_device_block(sdev);
}

static int
target_block(struct device *dev, void *data)
{
	if (scsi_is_target_device(dev))
		starget_for_each_device(to_scsi_target(dev), NULL,
					device_block);
	return 0;
}

void
scsi_target_block(struct device *dev)
{
	if (scsi_is_target_device(dev))
		starget_for_each_device(to_scsi_target(dev), NULL,
					device_block);
	else
		device_for_each_child(dev, NULL, target_block);
}
EXPORT_SYMBOL_GPL(scsi_target_block);

static void
device_unblock(struct scsi_device *sdev, void *data)
{
2530
	scsi_internal_device_unblock(sdev, *(enum scsi_device_state *)data);
Linus Torvalds's avatar
Linus Torvalds committed
2531 2532 2533 2534 2535 2536
}

static int
target_unblock(struct device *dev, void *data)
{
	if (scsi_is_target_device(dev))
2537
		starget_for_each_device(to_scsi_target(dev), data,
Linus Torvalds's avatar
Linus Torvalds committed
2538 2539 2540 2541 2542
					device_unblock);
	return 0;
}

void
2543
scsi_target_unblock(struct device *dev, enum scsi_device_state new_state)
Linus Torvalds's avatar
Linus Torvalds committed
2544 2545
{
	if (scsi_is_target_device(dev))
2546
		starget_for_each_device(to_scsi_target(dev), &new_state,
Linus Torvalds's avatar
Linus Torvalds committed
2547 2548
					device_unblock);
	else
2549
		device_for_each_child(dev, &new_state, target_unblock);
Linus Torvalds's avatar
Linus Torvalds committed
2550 2551
}
EXPORT_SYMBOL_GPL(scsi_target_unblock);
2552 2553 2554

/**
 * scsi_kmap_atomic_sg - find and atomically map an sg-elemnt
2555
 * @sgl:	scatter-gather list
2556 2557 2558 2559 2560 2561
 * @sg_count:	number of segments in sg
 * @offset:	offset in bytes into sg, on return offset into the mapped area
 * @len:	bytes to map, on return number of bytes mapped
 *
 * Returns virtual address of the start of the mapped page
 */
2562
void *scsi_kmap_atomic_sg(struct scatterlist *sgl, int sg_count,
2563 2564 2565 2566
			  size_t *offset, size_t *len)
{
	int i;
	size_t sg_len = 0, len_complete = 0;
2567
	struct scatterlist *sg;
2568 2569
	struct page *page;

2570 2571
	WARN_ON(!irqs_disabled());

2572
	for_each_sg(sgl, sg, sg_count, i) {
2573
		len_complete = sg_len; /* Complete sg-entries */
2574
		sg_len += sg->length;
2575 2576 2577 2578 2579
		if (sg_len > *offset)
			break;
	}

	if (unlikely(i == sg_count)) {
2580 2581
		printk(KERN_ERR "%s: Bytes in sg: %zu, requested offset %zu, "
			"elements %d\n",
2582
		       __func__, sg_len, *offset, sg_count);
2583 2584 2585 2586 2587
		WARN_ON(1);
		return NULL;
	}

	/* Offset starting from the beginning of first page in this sg-entry */
2588
	*offset = *offset - len_complete + sg->offset;
2589 2590

	/* Assumption: contiguous pages can be accessed as "page + i" */
2591
	page = nth_page(sg_page(sg), (*offset >> PAGE_SHIFT));
2592 2593 2594 2595 2596 2597 2598
	*offset &= ~PAGE_MASK;

	/* Bytes in this sg-entry from *offset to the end of the page */
	sg_len = PAGE_SIZE - *offset;
	if (*len > sg_len)
		*len = sg_len;

2599
	return kmap_atomic(page);
2600 2601 2602 2603
}
EXPORT_SYMBOL(scsi_kmap_atomic_sg);

/**
2604
 * scsi_kunmap_atomic_sg - atomically unmap a virtual address, previously mapped with scsi_kmap_atomic_sg
2605 2606 2607 2608
 * @virt:	virtual address to be unmapped
 */
void scsi_kunmap_atomic_sg(void *virt)
{
2609
	kunmap_atomic(virt);
2610 2611
}
EXPORT_SYMBOL(scsi_kunmap_atomic_sg);
2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625

void sdev_disable_disk_events(struct scsi_device *sdev)
{
	atomic_inc(&sdev->disk_events_disable_depth);
}
EXPORT_SYMBOL(sdev_disable_disk_events);

void sdev_enable_disk_events(struct scsi_device *sdev)
{
	if (WARN_ON_ONCE(atomic_read(&sdev->disk_events_disable_depth) <= 0))
		return;
	atomic_dec(&sdev->disk_events_disable_depth);
}
EXPORT_SYMBOL(sdev_enable_disk_events);