s390/dasd: Handle out-of-space constraint

The storage server issues three different types of out-of-space messages
whenever the Extent Pool or Extent Repository space runs short. When a
configured warning watermark is reached, the physical space is
completeley exhausted, or the capacity constraints have been relieved, a
message is received.

A log entry for the sysadmin to react to is generated in any case. In
case the physical space is completely exhausted, sense data that reads
"no space left on device" is received. In this case, currently running
I/O will be blocked until space has either been released or added to the
extent pool, and a relieve message was received via an attention
interrupt.
Signed-off-by: Jan Höppner <hoeppner@linux.ibm.com>
Reviewed-by: Stefan Haberland <sth@linux.ibm.com>
Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>

drivers/s390/block/dasd.c

@@ -1659,6 +1659,14 @@ static int dasd_ese_needs_format(struct dasd_block *block, struct irb *irb)
 		scsw_cstat(&irb->scsw) == SCHN_STAT_INCORR_LEN;
 }
 
+static int dasd_ese_oos_cond(u8 *sense)
+{
+	return sense[0] & SNS0_EQUIPMENT_CHECK &&
+		sense[1] & SNS1_PERM_ERR &&
+		sense[1] & SNS1_WRITE_INHIBITED &&
+		sense[25] == 0x01;
+}
+
 /*
  * Interrupt handler for "normal" ssch-io based dasd devices.
  */
@@ -1727,6 +1735,17 @@ void dasd_int_handler(struct ccw_device *cdev, unsigned long intparm,
 				test_bit(DASD_CQR_SUPPRESS_FP, &cqr->flags);
 			nrf_suppressed = (sense[1] & SNS1_NO_REC_FOUND) &&
 				test_bit(DASD_CQR_SUPPRESS_NRF, &cqr->flags);
+
+			/*
+			 * Extent pool probably out-of-space.
+			 * Stop device and check exhaust level.
+			 */
+			if (dasd_ese_oos_cond(sense)) {
+				dasd_generic_space_exhaust(device, cqr);
+				device->discipline->ext_pool_exhaust(device, cqr);
+				dasd_put_device(device);
+				return;
+			}
 		}
 		if (!(fp_suppressed || nrf_suppressed))
 			device->discipline->dump_sense_dbf(device, irb, "int");
@@ -2021,7 +2040,7 @@ static void __dasd_device_check_expire(struct dasd_device *device)
 static int __dasd_device_is_unusable(struct dasd_device *device,
 				     struct dasd_ccw_req *cqr)
 {
-	int mask = ~(DASD_STOPPED_DC_WAIT | DASD_UNRESUMED_PM);
+	int mask = ~(DASD_STOPPED_DC_WAIT | DASD_UNRESUMED_PM | DASD_STOPPED_NOSPC);
 
 	if (test_bit(DASD_FLAG_OFFLINE, &device->flags) &&
 	    !test_bit(DASD_FLAG_SAFE_OFFLINE_RUNNING, &device->flags)) {
@@ -3877,6 +3896,43 @@ int dasd_generic_verify_path(struct dasd_device *device, __u8 lpm)
 }
 EXPORT_SYMBOL_GPL(dasd_generic_verify_path);
 
+void dasd_generic_space_exhaust(struct dasd_device *device,
+				struct dasd_ccw_req *cqr)
+{
+	dasd_eer_write(device, NULL, DASD_EER_NOSPC);
+
+	if (device->state < DASD_STATE_BASIC)
+		return;
+
+	if (cqr->status == DASD_CQR_IN_IO ||
+	    cqr->status == DASD_CQR_CLEAR_PENDING) {
+		cqr->status = DASD_CQR_QUEUED;
+		cqr->retries++;
+	}
+	dasd_device_set_stop_bits(device, DASD_STOPPED_NOSPC);
+	dasd_device_clear_timer(device);
+	dasd_schedule_device_bh(device);
+}
+EXPORT_SYMBOL_GPL(dasd_generic_space_exhaust);
+
+void dasd_generic_space_avail(struct dasd_device *device)
+{
+	dev_info(&device->cdev->dev, "Extent pool space is available\n");
+	DBF_DEV_EVENT(DBF_WARNING, device, "%s", "space available");
+
+	dasd_device_remove_stop_bits(device, DASD_STOPPED_NOSPC);
+	dasd_schedule_device_bh(device);
+
+	if (device->block) {
+		dasd_schedule_block_bh(device->block);
+		if (device->block->request_queue)
+			blk_mq_run_hw_queues(device->block->request_queue, true);
+	}
+	if (!device->stopped)
+		wake_up(&generic_waitq);
+}
+EXPORT_SYMBOL_GPL(dasd_generic_space_avail);
+
 /*
  * clear active requests and requeue them to block layer if possible
  */

drivers/s390/block/dasd_eckd.c

@@ -89,6 +89,19 @@ static struct {
 } *dasd_reserve_req;
 static DEFINE_MUTEX(dasd_reserve_mutex);
 
+static struct {
+	struct dasd_ccw_req cqr;
+	struct ccw1 ccw[2];
+	char data[40];
+} *dasd_vol_info_req;
+static DEFINE_MUTEX(dasd_vol_info_mutex);
+
+struct ext_pool_exhaust_work_data {
+	struct work_struct worker;
+	struct dasd_device *device;
+	struct dasd_device *base;
+};
+
 /* definitions for the path verification worker */
 struct path_verification_work_data {
 	struct work_struct worker;
@@ -1479,6 +1492,7 @@ static int dasd_eckd_read_vol_info(struct dasd_device *device)
 	struct dasd_rssd_vsq *vsq;
 	struct dasd_ccw_req *cqr;
 	struct ccw1 *ccw;
+	int useglobal;
 	int rc;
 
 	/* This command cannot be executed on an alias device */
@@ -1486,12 +1500,20 @@ static int dasd_eckd_read_vol_info(struct dasd_device *device)
 	    private->uid.type == UA_HYPER_PAV_ALIAS)
 		return 0;
 
+	useglobal = 0;
 	cqr = dasd_smalloc_request(DASD_ECKD_MAGIC, 2 /* PSF + RSSD */,
 				   sizeof(*prssdp) + sizeof(*vsq), device, NULL);
 	if (IS_ERR(cqr)) {
 		DBF_EVENT_DEVID(DBF_WARNING, device->cdev, "%s",
 				"Could not allocate initialization request");
-		return PTR_ERR(cqr);
+		mutex_lock(&dasd_vol_info_mutex);
+		useglobal = 1;
+		cqr = &dasd_vol_info_req->cqr;
+		memset(cqr, 0, sizeof(*cqr));
+		memset(dasd_vol_info_req, 0, sizeof(*dasd_vol_info_req));
+		cqr->cpaddr = &dasd_vol_info_req->ccw;
+		cqr->data = &dasd_vol_info_req->data;
+		cqr->magic = DASD_ECKD_MAGIC;
 	}
 
 	/* Prepare for Read Subsystem Data */
@@ -1535,7 +1557,10 @@ static int dasd_eckd_read_vol_info(struct dasd_device *device)
 			 "Reading the volume storage information failed with rc=%d\n", rc);
 	}
 
-	dasd_sfree_request(cqr, cqr->memdev);
+	if (useglobal)
+		mutex_unlock(&dasd_vol_info_mutex);
+	else
+		dasd_sfree_request(cqr, cqr->memdev);
 
 	return rc;
 }
@@ -1590,6 +1615,53 @@ static int dasd_eckd_logical_capacity(struct dasd_device *device)
 	return private->vsq.logical_capacity;
 }
 
+static void dasd_eckd_ext_pool_exhaust_work(struct work_struct *work)
+{
+	struct ext_pool_exhaust_work_data *data;
+	struct dasd_device *device;
+	struct dasd_device *base;
+
+	data = container_of(work, struct ext_pool_exhaust_work_data, worker);
+	device = data->device;
+	base = data->base;
+
+	if (!base)
+		base = device;
+	if (dasd_eckd_space_configured(base) != 0) {
+		dasd_generic_space_avail(device);
+	} else {
+		dev_warn(&device->cdev->dev, "No space left in the extent pool\n");
+		DBF_DEV_EVENT(DBF_WARNING, device, "%s", "out of space");
+	}
+
+	dasd_put_device(device);
+	kfree(data);
+}
+
+static int dasd_eckd_ext_pool_exhaust(struct dasd_device *device,
+				      struct dasd_ccw_req *cqr)
+{
+	struct ext_pool_exhaust_work_data *data;
+
+	data = kzalloc(sizeof(*data), GFP_ATOMIC);
+	if (!data)
+		return -ENOMEM;
+	INIT_WORK(&data->worker, dasd_eckd_ext_pool_exhaust_work);
+	dasd_get_device(device);
+	data->device = device;
+
+	if (cqr->block)
+		data->base = cqr->block->base;
+	else if (cqr->basedev)
+		data->base = cqr->basedev;
+	else
+		data->base = NULL;
+
+	schedule_work(&data->worker);
+
+	return 0;
+}
+
 static void dasd_eckd_cpy_ext_pool_data(struct dasd_device *device,
 					struct dasd_rssd_lcq *lcq)
 {
@@ -2099,6 +2171,9 @@ dasd_eckd_analysis_ccw(struct dasd_device *device)
 	cqr->retries = 255;
 	cqr->buildclk = get_tod_clock();
 	cqr->status = DASD_CQR_FILLED;
+	/* Set flags to suppress output for expected errors */
+	set_bit(DASD_CQR_SUPPRESS_NRF, &cqr->flags);
+
 	return cqr;
 }
 
@@ -6267,6 +6342,73 @@ static void dasd_eckd_handle_cuir(struct dasd_device *device, void *messages,
 	device->discipline->check_attention(device, lpum);
 }
 
+static void dasd_eckd_oos_resume(struct dasd_device *device)
+{
+	struct dasd_eckd_private *private = device->private;
+	struct alias_pav_group *pavgroup, *tempgroup;
+	struct dasd_device *dev, *n;
+	unsigned long flags;
+
+	spin_lock_irqsave(&private->lcu->lock, flags);
+	list_for_each_entry_safe(dev, n, &private->lcu->active_devices,
+				 alias_list) {
+		if (dev->stopped & DASD_STOPPED_NOSPC)
+			dasd_generic_space_avail(dev);
+	}
+	list_for_each_entry_safe(dev, n, &private->lcu->inactive_devices,
+				 alias_list) {
+		if (dev->stopped & DASD_STOPPED_NOSPC)
+			dasd_generic_space_avail(dev);
+	}
+	/* devices in PAV groups */
+	list_for_each_entry_safe(pavgroup, tempgroup,
+				 &private->lcu->grouplist,
+				 group) {
+		list_for_each_entry_safe(dev, n, &pavgroup->baselist,
+					 alias_list) {
+			if (dev->stopped & DASD_STOPPED_NOSPC)
+				dasd_generic_space_avail(dev);
+		}
+		list_for_each_entry_safe(dev, n, &pavgroup->aliaslist,
+					 alias_list) {
+			if (dev->stopped & DASD_STOPPED_NOSPC)
+				dasd_generic_space_avail(dev);
+		}
+	}
+	spin_unlock_irqrestore(&private->lcu->lock, flags);
+}
+
+static void dasd_eckd_handle_oos(struct dasd_device *device, void *messages,
+				 __u8 lpum)
+{
+	struct dasd_oos_message *oos = messages;
+
+	switch (oos->code) {
+	case REPO_WARN:
+	case POOL_WARN:
+		dev_warn(&device->cdev->dev,
+			 "Extent pool usage has reached a critical value\n");
+		dasd_eckd_oos_resume(device);
+		break;
+	case REPO_EXHAUST:
+	case POOL_EXHAUST:
+		dev_warn(&device->cdev->dev,
+			 "Extent pool is exhausted\n");
+		break;
+	case REPO_RELIEVE:
+	case POOL_RELIEVE:
+		dev_info(&device->cdev->dev,
+			 "Extent pool physical space constraint has been relieved\n");
+		break;
+	}
+
+	/* In any case, update related data */
+	dasd_eckd_read_ext_pool_info(device);
+
+	/* to make sure there is no attention left schedule work again */
+	device->discipline->check_attention(device, lpum);
+}
+
 static void dasd_eckd_check_attention_work(struct work_struct *work)
 {
 	struct check_attention_work_data *data;
@@ -6285,9 +6427,14 @@ static void dasd_eckd_check_attention_work(struct work_struct *work)
 	rc = dasd_eckd_read_message_buffer(device, messages, data->lpum);
 	if (rc)
 		goto out;
+
 	if (messages->length == ATTENTION_LENGTH_CUIR &&
 	    messages->format == ATTENTION_FORMAT_CUIR)
 		dasd_eckd_handle_cuir(device, messages, data->lpum);
+	if (messages->length == ATTENTION_LENGTH_OOS &&
+	    messages->format == ATTENTION_FORMAT_OOS)
+		dasd_eckd_handle_oos(device, messages, data->lpum);
+
 out:
 	dasd_put_device(device);
 	kfree(messages);
@@ -6501,6 +6648,7 @@ static struct dasd_discipline dasd_eckd_discipline = {
 	.ext_pool_cap_at_warnlevel = dasd_eckd_ext_pool_cap_at_warnlevel,
 	.ext_pool_warn_thrshld = dasd_eckd_ext_pool_warn_thrshld,
 	.ext_pool_oos = dasd_eckd_ext_pool_oos,
+	.ext_pool_exhaust = dasd_eckd_ext_pool_exhaust,
 	.ese_format = dasd_eckd_ese_format,
 	.ese_read = dasd_eckd_ese_read,
 };
@@ -6515,16 +6663,22 @@ dasd_eckd_init(void)
 				   GFP_KERNEL | GFP_DMA);
 	if (!dasd_reserve_req)
 		return -ENOMEM;
+	dasd_vol_info_req = kmalloc(sizeof(*dasd_vol_info_req),
+				    GFP_KERNEL | GFP_DMA);
+	if (!dasd_vol_info_req)
+		return -ENOMEM;
 	path_verification_worker = kmalloc(sizeof(*path_verification_worker),
 				   GFP_KERNEL | GFP_DMA);
 	if (!path_verification_worker) {
 		kfree(dasd_reserve_req);
+		kfree(dasd_vol_info_req);
 		return -ENOMEM;
 	}
 	rawpadpage = (void *)__get_free_page(GFP_KERNEL);
 	if (!rawpadpage) {
 		kfree(path_verification_worker);
 		kfree(dasd_reserve_req);
+		kfree(dasd_vol_info_req);
 		return -ENOMEM;
 	}
 	ret = ccw_driver_register(&dasd_eckd_driver);
@@ -6533,6 +6687,7 @@ dasd_eckd_init(void)
 	else {
 		kfree(path_verification_worker);
 		kfree(dasd_reserve_req);
+		kfree(dasd_vol_info_req);
 		free_page((unsigned long)rawpadpage);
 	}
 	return ret;

drivers/s390/block/dasd_eckd.h

@@ -89,11 +89,23 @@
 #define CUIR_QUIESCE			 0x01
 #define CUIR_RESUME			 0x02
 
+/*
+ * Out-of-space (OOS) Codes
+ */
+#define REPO_WARN			 0x01
+#define REPO_EXHAUST			 0x02
+#define POOL_WARN			 0x03
+#define POOL_EXHAUST			 0x04
+#define REPO_RELIEVE			 0x05
+#define POOL_RELIEVE			 0x06
+
 /*
  * attention message definitions
  */
 #define ATTENTION_LENGTH_CUIR		 0x0e
 #define ATTENTION_FORMAT_CUIR		 0x01
+#define ATTENTION_LENGTH_OOS		 0x10
+#define ATTENTION_FORMAT_OOS		 0x06
 
 #define DASD_ECKD_PG_GROUPED		 0x10
 
@@ -449,6 +461,17 @@ struct dasd_rssd_lcq {
 	struct dasd_ext_pool_sum ext_pool_sum[448];
 } __packed;
 
+struct dasd_oos_message {
+	__u16 length;
+	__u8 format;
+	__u8 code;
+	__u8 percentage_empty;
+	__u8 reserved;
+	__u16 ext_pool_id;
+	__u16 token;
+	__u8 unused[6];
+} __packed;
+
 struct dasd_cuir_message {
 	__u16 length;
 	__u8 format;

drivers/s390/block/dasd_eer.c

@@ -386,6 +386,7 @@ void dasd_eer_write(struct dasd_device *device, struct dasd_ccw_req *cqr,
 		dasd_eer_write_standard_trigger(device, cqr, id);
 		break;
 	case DASD_EER_NOPATH:
+	case DASD_EER_NOSPC:
 		dasd_eer_write_standard_trigger(device, NULL, id);
 		break;
 	case DASD_EER_STATECHANGE:

drivers/s390/block/dasd_int.h

@@ -386,6 +386,7 @@ struct dasd_discipline {
 	int (*ext_pool_cap_at_warnlevel)(struct dasd_device *);
 	int (*ext_pool_warn_thrshld)(struct dasd_device *);
 	int (*ext_pool_oos)(struct dasd_device *);
+	int (*ext_pool_exhaust)(struct dasd_device *, struct dasd_ccw_req *);
 	struct dasd_ccw_req *(*ese_format)(struct dasd_device *, struct dasd_ccw_req *);
 	void (*ese_read)(struct dasd_ccw_req *);
 };
@@ -407,6 +408,7 @@ extern struct dasd_discipline *dasd_diag_discipline_pointer;
 #define DASD_EER_NOPATH      2
 #define DASD_EER_STATECHANGE 3
 #define DASD_EER_PPRCSUSPEND 4
+#define DASD_EER_NOSPC	     5
 
 /* DASD path handling */
 
@@ -581,6 +583,7 @@ struct dasd_queue {
 #define DASD_STOPPED_SU      16        /* summary unit check handling */
 #define DASD_STOPPED_PM      32        /* pm state transition */
 #define DASD_UNRESUMED_PM    64        /* pm resume failed state */
+#define DASD_STOPPED_NOSPC   128       /* no space left */
 
 /* per device flags */
 #define DASD_FLAG_OFFLINE	3	/* device is in offline processing */
@@ -776,6 +779,8 @@ int dasd_generic_restore_device(struct ccw_device *);
 enum uc_todo dasd_generic_uc_handler(struct ccw_device *, struct irb *);
 void dasd_generic_path_event(struct ccw_device *, int *);
 int dasd_generic_verify_path(struct dasd_device *, __u8);
+void dasd_generic_space_exhaust(struct dasd_device *, struct dasd_ccw_req *);
+void dasd_generic_space_avail(struct dasd_device *);
 
 int dasd_generic_read_dev_chars(struct dasd_device *, int, void *, int);
 char *dasd_get_sense(struct irb *);