zoned.c 60.5 KB
Newer Older
1 2
// SPDX-License-Identifier: GPL-2.0

3
#include <linux/bitops.h>
4 5
#include <linux/slab.h>
#include <linux/blkdev.h>
6
#include <linux/sched/mm.h>
7
#include <linux/atomic.h>
8
#include <linux/vmalloc.h>
9 10 11 12
#include "ctree.h"
#include "volumes.h"
#include "zoned.h"
#include "rcu-string.h"
13
#include "disk-io.h"
14
#include "block-group.h"
15
#include "transaction.h"
16
#include "dev-replace.h"
17
#include "space-info.h"
18
#include "super.h"
19
#include "fs.h"
20
#include "accessors.h"
21
#include "bio.h"
22 23 24

/* Maximum number of zones to report per blkdev_report_zones() call */
#define BTRFS_REPORT_NR_ZONES   4096
25 26 27 28
/* Invalid allocation pointer value for missing devices */
#define WP_MISSING_DEV ((u64)-1)
/* Pseudo write pointer value for conventional zone */
#define WP_CONVENTIONAL ((u64)-2)
29

30 31 32 33 34 35 36 37 38 39 40 41 42 43
/*
 * Location of the first zone of superblock logging zone pairs.
 *
 * - primary superblock:    0B (zone 0)
 * - first copy:          512G (zone starting at that offset)
 * - second copy:           4T (zone starting at that offset)
 */
#define BTRFS_SB_LOG_PRIMARY_OFFSET	(0ULL)
#define BTRFS_SB_LOG_FIRST_OFFSET	(512ULL * SZ_1G)
#define BTRFS_SB_LOG_SECOND_OFFSET	(4096ULL * SZ_1G)

#define BTRFS_SB_LOG_FIRST_SHIFT	const_ilog2(BTRFS_SB_LOG_FIRST_OFFSET)
#define BTRFS_SB_LOG_SECOND_SHIFT	const_ilog2(BTRFS_SB_LOG_SECOND_OFFSET)

44 45 46
/* Number of superblock log zones */
#define BTRFS_NR_SB_LOG_ZONES 2

47 48 49 50 51 52 53 54 55 56
/*
 * Minimum of active zones we need:
 *
 * - BTRFS_SUPER_MIRROR_MAX zones for superblock mirrors
 * - 3 zones to ensure at least one zone per SYSTEM, META and DATA block group
 * - 1 zone for tree-log dedicated block group
 * - 1 zone for relocation
 */
#define BTRFS_MIN_ACTIVE_ZONES		(BTRFS_SUPER_MIRROR_MAX + 5)

57
/*
58 59 60 61
 * Minimum / maximum supported zone size. Currently, SMR disks have a zone
 * size of 256MiB, and we are expecting ZNS drives to be in the 1-4GiB range.
 * We do not expect the zone size to become larger than 8GiB or smaller than
 * 4MiB in the near future.
62 63
 */
#define BTRFS_MAX_ZONE_SIZE		SZ_8G
64
#define BTRFS_MIN_ZONE_SIZE		SZ_4M
65

66 67 68 69 70 71 72 73
#define SUPER_INFO_SECTORS	((u64)BTRFS_SUPER_INFO_SIZE >> SECTOR_SHIFT)

static inline bool sb_zone_is_full(const struct blk_zone *zone)
{
	return (zone->cond == BLK_ZONE_COND_FULL) ||
		(zone->wp + SUPER_INFO_SECTORS > zone->start + zone->capacity);
}

74 75 76 77 78 79 80 81 82
static int copy_zone_info_cb(struct blk_zone *zone, unsigned int idx, void *data)
{
	struct blk_zone *zones = data;

	memcpy(&zones[idx], zone, sizeof(*zone));

	return 0;
}

83 84 85 86 87 88
static int sb_write_pointer(struct block_device *bdev, struct blk_zone *zones,
			    u64 *wp_ret)
{
	bool empty[BTRFS_NR_SB_LOG_ZONES];
	bool full[BTRFS_NR_SB_LOG_ZONES];
	sector_t sector;
89
	int i;
90

91 92 93 94 95
	for (i = 0; i < BTRFS_NR_SB_LOG_ZONES; i++) {
		ASSERT(zones[i].type != BLK_ZONE_TYPE_CONVENTIONAL);
		empty[i] = (zones[i].cond == BLK_ZONE_COND_EMPTY);
		full[i] = sb_zone_is_full(&zones[i]);
	}
96 97 98 99 100

	/*
	 * Possible states of log buffer zones
	 *
	 *           Empty[0]  In use[0]  Full[0]
101 102 103
	 * Empty[1]         *          0        1
	 * In use[1]        x          x        1
	 * Full[1]          0          0        C
104 105 106 107 108
	 *
	 * Log position:
	 *   *: Special case, no superblock is written
	 *   0: Use write pointer of zones[0]
	 *   1: Use write pointer of zones[1]
109
	 *   C: Compare super blocks from zones[0] and zones[1], use the latest
110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125
	 *      one determined by generation
	 *   x: Invalid state
	 */

	if (empty[0] && empty[1]) {
		/* Special case to distinguish no superblock to read */
		*wp_ret = zones[0].start << SECTOR_SHIFT;
		return -ENOENT;
	} else if (full[0] && full[1]) {
		/* Compare two super blocks */
		struct address_space *mapping = bdev->bd_inode->i_mapping;
		struct page *page[BTRFS_NR_SB_LOG_ZONES];
		struct btrfs_super_block *super[BTRFS_NR_SB_LOG_ZONES];
		int i;

		for (i = 0; i < BTRFS_NR_SB_LOG_ZONES; i++) {
126 127 128
			u64 zone_end = (zones[i].start + zones[i].capacity) << SECTOR_SHIFT;
			u64 bytenr = ALIGN_DOWN(zone_end, BTRFS_SUPER_INFO_SIZE) -
						BTRFS_SUPER_INFO_SIZE;
129 130 131 132 133 134 135 136 137 138 139

			page[i] = read_cache_page_gfp(mapping,
					bytenr >> PAGE_SHIFT, GFP_NOFS);
			if (IS_ERR(page[i])) {
				if (i == 1)
					btrfs_release_disk_super(super[0]);
				return PTR_ERR(page[i]);
			}
			super[i] = page_address(page[i]);
		}

140 141
		if (btrfs_super_generation(super[0]) >
		    btrfs_super_generation(super[1]))
142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159
			sector = zones[1].start;
		else
			sector = zones[0].start;

		for (i = 0; i < BTRFS_NR_SB_LOG_ZONES; i++)
			btrfs_release_disk_super(super[i]);
	} else if (!full[0] && (empty[1] || full[1])) {
		sector = zones[0].wp;
	} else if (full[0]) {
		sector = zones[1].wp;
	} else {
		return -EUCLEAN;
	}
	*wp_ret = sector << SECTOR_SHIFT;
	return 0;
}

/*
160
 * Get the first zone number of the superblock mirror
161 162 163
 */
static inline u32 sb_zone_number(int shift, int mirror)
{
164
	u64 zone = U64_MAX;
165

166
	ASSERT(mirror < BTRFS_SUPER_MIRROR_MAX);
167
	switch (mirror) {
168 169 170
	case 0: zone = 0; break;
	case 1: zone = 1ULL << (BTRFS_SB_LOG_FIRST_SHIFT - shift); break;
	case 2: zone = 1ULL << (BTRFS_SB_LOG_SECOND_SHIFT - shift); break;
171 172
	}

173 174 175
	ASSERT(zone <= U32_MAX);

	return (u32)zone;
176 177
}

178 179 180 181 182 183 184 185 186 187 188 189
static inline sector_t zone_start_sector(u32 zone_number,
					 struct block_device *bdev)
{
	return (sector_t)zone_number << ilog2(bdev_zone_sectors(bdev));
}

static inline u64 zone_start_physical(u32 zone_number,
				      struct btrfs_zoned_device_info *zone_info)
{
	return (u64)zone_number << zone_info->zone_size_shift;
}

190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219
/*
 * Emulate blkdev_report_zones() for a non-zoned device. It slices up the block
 * device into static sized chunks and fake a conventional zone on each of
 * them.
 */
static int emulate_report_zones(struct btrfs_device *device, u64 pos,
				struct blk_zone *zones, unsigned int nr_zones)
{
	const sector_t zone_sectors = device->fs_info->zone_size >> SECTOR_SHIFT;
	sector_t bdev_size = bdev_nr_sectors(device->bdev);
	unsigned int i;

	pos >>= SECTOR_SHIFT;
	for (i = 0; i < nr_zones; i++) {
		zones[i].start = i * zone_sectors + pos;
		zones[i].len = zone_sectors;
		zones[i].capacity = zone_sectors;
		zones[i].wp = zones[i].start + zone_sectors;
		zones[i].type = BLK_ZONE_TYPE_CONVENTIONAL;
		zones[i].cond = BLK_ZONE_COND_NOT_WP;

		if (zones[i].wp >= bdev_size) {
			i++;
			break;
		}
	}

	return i;
}

220 221 222
static int btrfs_get_dev_zones(struct btrfs_device *device, u64 pos,
			       struct blk_zone *zones, unsigned int *nr_zones)
{
223
	struct btrfs_zoned_device_info *zinfo = device->zone_info;
224 225 226 227 228
	int ret;

	if (!*nr_zones)
		return 0;

229 230 231 232 233 234
	if (!bdev_is_zoned(device->bdev)) {
		ret = emulate_report_zones(device, pos, zones, *nr_zones);
		*nr_zones = ret;
		return 0;
	}

235 236 237
	/* Check cache */
	if (zinfo->zone_cache) {
		unsigned int i;
238
		u32 zno;
239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263

		ASSERT(IS_ALIGNED(pos, zinfo->zone_size));
		zno = pos >> zinfo->zone_size_shift;
		/*
		 * We cannot report zones beyond the zone end. So, it is OK to
		 * cap *nr_zones to at the end.
		 */
		*nr_zones = min_t(u32, *nr_zones, zinfo->nr_zones - zno);

		for (i = 0; i < *nr_zones; i++) {
			struct blk_zone *zone_info;

			zone_info = &zinfo->zone_cache[zno + i];
			if (!zone_info->len)
				break;
		}

		if (i == *nr_zones) {
			/* Cache hit on all the zones */
			memcpy(zones, zinfo->zone_cache + zno,
			       sizeof(*zinfo->zone_cache) * *nr_zones);
			return 0;
		}
	}

264 265 266 267 268 269 270 271 272 273 274 275 276
	ret = blkdev_report_zones(device->bdev, pos >> SECTOR_SHIFT, *nr_zones,
				  copy_zone_info_cb, zones);
	if (ret < 0) {
		btrfs_err_in_rcu(device->fs_info,
				 "zoned: failed to read zone %llu on %s (devid %llu)",
				 pos, rcu_str_deref(device->name),
				 device->devid);
		return ret;
	}
	*nr_zones = ret;
	if (!ret)
		return -EIO;

277
	/* Populate cache */
278 279 280
	if (zinfo->zone_cache) {
		u32 zno = pos >> zinfo->zone_size_shift;

281 282
		memcpy(zinfo->zone_cache + zno, zones,
		       sizeof(*zinfo->zone_cache) * *nr_zones);
283
	}
284

285 286 287
	return 0;
}

288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310
/* The emulated zone size is determined from the size of device extent */
static int calculate_emulated_zone_size(struct btrfs_fs_info *fs_info)
{
	struct btrfs_path *path;
	struct btrfs_root *root = fs_info->dev_root;
	struct btrfs_key key;
	struct extent_buffer *leaf;
	struct btrfs_dev_extent *dext;
	int ret = 0;

	key.objectid = 1;
	key.type = BTRFS_DEV_EXTENT_KEY;
	key.offset = 0;

	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
	if (ret < 0)
		goto out;

	if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) {
311
		ret = btrfs_next_leaf(root, path);
312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331
		if (ret < 0)
			goto out;
		/* No dev extents at all? Not good */
		if (ret > 0) {
			ret = -EUCLEAN;
			goto out;
		}
	}

	leaf = path->nodes[0];
	dext = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_extent);
	fs_info->zone_size = btrfs_dev_extent_length(leaf, dext);
	ret = 0;

out:
	btrfs_free_path(path);

	return ret;
}

332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347
int btrfs_get_dev_zone_info_all_devices(struct btrfs_fs_info *fs_info)
{
	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
	struct btrfs_device *device;
	int ret = 0;

	/* fs_info->zone_size might not set yet. Use the incomapt flag here. */
	if (!btrfs_fs_incompat(fs_info, ZONED))
		return 0;

	mutex_lock(&fs_devices->device_list_mutex);
	list_for_each_entry(device, &fs_devices->devices, dev_list) {
		/* We can skip reading of zone info for missing devices */
		if (!device->bdev)
			continue;

348
		ret = btrfs_get_dev_zone_info(device, true);
349 350 351 352 353 354 355 356
		if (ret)
			break;
	}
	mutex_unlock(&fs_devices->device_list_mutex);

	return ret;
}

357
int btrfs_get_dev_zone_info(struct btrfs_device *device, bool populate_cache)
358
{
359
	struct btrfs_fs_info *fs_info = device->fs_info;
360 361
	struct btrfs_zoned_device_info *zone_info = NULL;
	struct block_device *bdev = device->bdev;
362 363
	unsigned int max_active_zones;
	unsigned int nactive;
364 365 366 367
	sector_t nr_sectors;
	sector_t sector = 0;
	struct blk_zone *zones = NULL;
	unsigned int i, nreported = 0, nr_zones;
368
	sector_t zone_sectors;
369
	char *model, *emulated;
370 371
	int ret;

372 373 374 375 376
	/*
	 * Cannot use btrfs_is_zoned here, since fs_info::zone_size might not
	 * yet be set.
	 */
	if (!btrfs_fs_incompat(fs_info, ZONED))
377 378 379 380 381 382 383 384 385
		return 0;

	if (device->zone_info)
		return 0;

	zone_info = kzalloc(sizeof(*zone_info), GFP_KERNEL);
	if (!zone_info)
		return -ENOMEM;

386 387
	device->zone_info = zone_info;

388 389 390 391 392 393 394 395 396 397 398 399 400
	if (!bdev_is_zoned(bdev)) {
		if (!fs_info->zone_size) {
			ret = calculate_emulated_zone_size(fs_info);
			if (ret)
				goto out;
		}

		ASSERT(fs_info->zone_size);
		zone_sectors = fs_info->zone_size >> SECTOR_SHIFT;
	} else {
		zone_sectors = bdev_zone_sectors(bdev);
	}

401
	ASSERT(is_power_of_two_u64(zone_sectors));
402
	zone_info->zone_size = zone_sectors << SECTOR_SHIFT;
403 404 405 406 407 408 409 410 411

	/* We reject devices with a zone size larger than 8GB */
	if (zone_info->zone_size > BTRFS_MAX_ZONE_SIZE) {
		btrfs_err_in_rcu(fs_info,
		"zoned: %s: zone size %llu larger than supported maximum %llu",
				 rcu_str_deref(device->name),
				 zone_info->zone_size, BTRFS_MAX_ZONE_SIZE);
		ret = -EINVAL;
		goto out;
412 413 414 415 416 417 418
	} else if (zone_info->zone_size < BTRFS_MIN_ZONE_SIZE) {
		btrfs_err_in_rcu(fs_info,
		"zoned: %s: zone size %llu smaller than supported minimum %u",
				 rcu_str_deref(device->name),
				 zone_info->zone_size, BTRFS_MIN_ZONE_SIZE);
		ret = -EINVAL;
		goto out;
419 420 421
	}

	nr_sectors = bdev_nr_sectors(bdev);
422 423 424 425 426
	zone_info->zone_size_shift = ilog2(zone_info->zone_size);
	zone_info->nr_zones = nr_sectors >> ilog2(zone_sectors);
	if (!IS_ALIGNED(nr_sectors, zone_sectors))
		zone_info->nr_zones++;

427
	max_active_zones = bdev_max_active_zones(bdev);
428 429 430 431 432 433 434 435 436 437
	if (max_active_zones && max_active_zones < BTRFS_MIN_ACTIVE_ZONES) {
		btrfs_err_in_rcu(fs_info,
"zoned: %s: max active zones %u is too small, need at least %u active zones",
				 rcu_str_deref(device->name), max_active_zones,
				 BTRFS_MIN_ACTIVE_ZONES);
		ret = -EINVAL;
		goto out;
	}
	zone_info->max_active_zones = max_active_zones;

438 439 440 441 442 443 444 445 446 447 448 449
	zone_info->seq_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL);
	if (!zone_info->seq_zones) {
		ret = -ENOMEM;
		goto out;
	}

	zone_info->empty_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL);
	if (!zone_info->empty_zones) {
		ret = -ENOMEM;
		goto out;
	}

450 451 452 453 454 455
	zone_info->active_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL);
	if (!zone_info->active_zones) {
		ret = -ENOMEM;
		goto out;
	}

456
	zones = kvcalloc(BTRFS_REPORT_NR_ZONES, sizeof(struct blk_zone), GFP_KERNEL);
457 458 459 460 461
	if (!zones) {
		ret = -ENOMEM;
		goto out;
	}

462 463 464 465 466 467
	/*
	 * Enable zone cache only for a zoned device. On a non-zoned device, we
	 * fill the zone info with emulated CONVENTIONAL zones, so no need to
	 * use the cache.
	 */
	if (populate_cache && bdev_is_zoned(device->bdev)) {
468 469
		zone_info->zone_cache = vcalloc(zone_info->nr_zones,
						sizeof(struct blk_zone));
470 471 472 473 474 475 476 477 478
		if (!zone_info->zone_cache) {
			btrfs_err_in_rcu(device->fs_info,
				"zoned: failed to allocate zone cache for %s",
				rcu_str_deref(device->name));
			ret = -ENOMEM;
			goto out;
		}
	}

479
	/* Get zones type */
480
	nactive = 0;
481 482 483 484 485 486 487 488 489 490
	while (sector < nr_sectors) {
		nr_zones = BTRFS_REPORT_NR_ZONES;
		ret = btrfs_get_dev_zones(device, sector << SECTOR_SHIFT, zones,
					  &nr_zones);
		if (ret)
			goto out;

		for (i = 0; i < nr_zones; i++) {
			if (zones[i].type == BLK_ZONE_TYPE_SEQWRITE_REQ)
				__set_bit(nreported, zone_info->seq_zones);
491 492
			switch (zones[i].cond) {
			case BLK_ZONE_COND_EMPTY:
493
				__set_bit(nreported, zone_info->empty_zones);
494 495 496 497 498 499 500 501
				break;
			case BLK_ZONE_COND_IMP_OPEN:
			case BLK_ZONE_COND_EXP_OPEN:
			case BLK_ZONE_COND_CLOSED:
				__set_bit(nreported, zone_info->active_zones);
				nactive++;
				break;
			}
502 503 504 505 506 507 508 509 510 511 512 513 514 515
			nreported++;
		}
		sector = zones[nr_zones - 1].start + zones[nr_zones - 1].len;
	}

	if (nreported != zone_info->nr_zones) {
		btrfs_err_in_rcu(device->fs_info,
				 "inconsistent number of zones on %s (%u/%u)",
				 rcu_str_deref(device->name), nreported,
				 zone_info->nr_zones);
		ret = -EIO;
		goto out;
	}

516 517 518 519 520 521 522 523 524 525 526
	if (max_active_zones) {
		if (nactive > max_active_zones) {
			btrfs_err_in_rcu(device->fs_info,
			"zoned: %u active zones on %s exceeds max_active_zones %u",
					 nactive, rcu_str_deref(device->name),
					 max_active_zones);
			ret = -EIO;
			goto out;
		}
		atomic_set(&zone_info->active_zones_left,
			   max_active_zones - nactive);
527
		set_bit(BTRFS_FS_ACTIVE_ZONE_TRACKING, &fs_info->flags);
528 529
	}

530 531 532 533 534 535 536 537 538 539 540
	/* Validate superblock log */
	nr_zones = BTRFS_NR_SB_LOG_ZONES;
	for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
		u32 sb_zone;
		u64 sb_wp;
		int sb_pos = BTRFS_NR_SB_LOG_ZONES * i;

		sb_zone = sb_zone_number(zone_info->zone_size_shift, i);
		if (sb_zone + 1 >= zone_info->nr_zones)
			continue;

541 542
		ret = btrfs_get_dev_zones(device,
					  zone_start_physical(sb_zone, zone_info),
543 544 545 546 547 548 549 550 551 552 553 554 555 556
					  &zone_info->sb_zones[sb_pos],
					  &nr_zones);
		if (ret)
			goto out;

		if (nr_zones != BTRFS_NR_SB_LOG_ZONES) {
			btrfs_err_in_rcu(device->fs_info,
	"zoned: failed to read super block log zone info at devid %llu zone %u",
					 device->devid, sb_zone);
			ret = -EUCLEAN;
			goto out;
		}

		/*
557
		 * If zones[0] is conventional, always use the beginning of the
558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575
		 * zone to record superblock. No need to validate in that case.
		 */
		if (zone_info->sb_zones[BTRFS_NR_SB_LOG_ZONES * i].type ==
		    BLK_ZONE_TYPE_CONVENTIONAL)
			continue;

		ret = sb_write_pointer(device->bdev,
				       &zone_info->sb_zones[sb_pos], &sb_wp);
		if (ret != -ENOENT && ret) {
			btrfs_err_in_rcu(device->fs_info,
			"zoned: super block log zone corrupted devid %llu zone %u",
					 device->devid, sb_zone);
			ret = -EUCLEAN;
			goto out;
		}
	}


576
	kvfree(zones);
577

578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603
	switch (bdev_zoned_model(bdev)) {
	case BLK_ZONED_HM:
		model = "host-managed zoned";
		emulated = "";
		break;
	case BLK_ZONED_HA:
		model = "host-aware zoned";
		emulated = "";
		break;
	case BLK_ZONED_NONE:
		model = "regular";
		emulated = "emulated ";
		break;
	default:
		/* Just in case */
		btrfs_err_in_rcu(fs_info, "zoned: unsupported model %d on %s",
				 bdev_zoned_model(bdev),
				 rcu_str_deref(device->name));
		ret = -EOPNOTSUPP;
		goto out_free_zone_info;
	}

	btrfs_info_in_rcu(fs_info,
		"%s block device %s, %u %szones of %llu bytes",
		model, rcu_str_deref(device->name), zone_info->nr_zones,
		emulated, zone_info->zone_size);
604 605 606 607

	return 0;

out:
608
	kvfree(zones);
609
out_free_zone_info:
610
	btrfs_destroy_dev_zone_info(device);
611 612 613 614 615 616 617 618 619 620 621

	return ret;
}

void btrfs_destroy_dev_zone_info(struct btrfs_device *device)
{
	struct btrfs_zoned_device_info *zone_info = device->zone_info;

	if (!zone_info)
		return;

622
	bitmap_free(zone_info->active_zones);
623 624
	bitmap_free(zone_info->seq_zones);
	bitmap_free(zone_info->empty_zones);
625
	vfree(zone_info->zone_cache);
626 627 628 629
	kfree(zone_info);
	device->zone_info = NULL;
}

630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669
struct btrfs_zoned_device_info *btrfs_clone_dev_zone_info(struct btrfs_device *orig_dev)
{
	struct btrfs_zoned_device_info *zone_info;

	zone_info = kmemdup(orig_dev->zone_info, sizeof(*zone_info), GFP_KERNEL);
	if (!zone_info)
		return NULL;

	zone_info->seq_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL);
	if (!zone_info->seq_zones)
		goto out;

	bitmap_copy(zone_info->seq_zones, orig_dev->zone_info->seq_zones,
		    zone_info->nr_zones);

	zone_info->empty_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL);
	if (!zone_info->empty_zones)
		goto out;

	bitmap_copy(zone_info->empty_zones, orig_dev->zone_info->empty_zones,
		    zone_info->nr_zones);

	zone_info->active_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL);
	if (!zone_info->active_zones)
		goto out;

	bitmap_copy(zone_info->active_zones, orig_dev->zone_info->active_zones,
		    zone_info->nr_zones);
	zone_info->zone_cache = NULL;

	return zone_info;

out:
	bitmap_free(zone_info->seq_zones);
	bitmap_free(zone_info->empty_zones);
	bitmap_free(zone_info->active_zones);
	kfree(zone_info);
	return NULL;
}

670 671 672 673 674 675 676 677 678 679 680 681
int btrfs_get_dev_zone(struct btrfs_device *device, u64 pos,
		       struct blk_zone *zone)
{
	unsigned int nr_zones = 1;
	int ret;

	ret = btrfs_get_dev_zones(device, pos, zone, &nr_zones);
	if (ret != 0 || !nr_zones)
		return ret ? ret : -EIO;

	return 0;
}
682

683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699
static int btrfs_check_for_zoned_device(struct btrfs_fs_info *fs_info)
{
	struct btrfs_device *device;

	list_for_each_entry(device, &fs_info->fs_devices->devices, dev_list) {
		if (device->bdev &&
		    bdev_zoned_model(device->bdev) == BLK_ZONED_HM) {
			btrfs_err(fs_info,
				"zoned: mode not enabled but zoned device found: %pg",
				device->bdev);
			return -EINVAL;
		}
	}

	return 0;
}

700 701
int btrfs_check_zoned_mode(struct btrfs_fs_info *fs_info)
{
702
	struct queue_limits *lim = &fs_info->limits;
703 704
	struct btrfs_device *device;
	u64 zone_size = 0;
705
	int ret;
706

707 708 709 710 711 712 713
	/*
	 * Host-Managed devices can't be used without the ZONED flag.  With the
	 * ZONED all devices can be used, using zone emulation if required.
	 */
	if (!btrfs_fs_incompat(fs_info, ZONED))
		return btrfs_check_for_zoned_device(fs_info);

714 715
	blk_set_stacking_limits(lim);

716 717
	list_for_each_entry(device, &fs_info->fs_devices->devices, dev_list) {
		struct btrfs_zoned_device_info *zone_info = device->zone_info;
718 719 720 721

		if (!device->bdev)
			continue;

722 723 724 725
		if (!zone_size) {
			zone_size = zone_info->zone_size;
		} else if (zone_info->zone_size != zone_size) {
			btrfs_err(fs_info,
726
		"zoned: unequal block device zone sizes: have %llu found %llu",
727 728
				  zone_info->zone_size, zone_size);
			return -EINVAL;
729
		}
730 731 732 733 734 735 736 737 738 739 740

		/*
		 * With the zoned emulation, we can have non-zoned device on the
		 * zoned mode. In this case, we don't have a valid max zone
		 * append size.
		 */
		if (bdev_is_zoned(device->bdev)) {
			blk_stack_limits(lim,
					 &bdev_get_queue(device->bdev)->limits,
					 0);
		}
741 742 743 744
	}

	/*
	 * stripe_size is always aligned to BTRFS_STRIPE_LEN in
745
	 * btrfs_create_chunk(). Since we want stripe_len == zone_size,
746 747 748 749 750 751
	 * check the alignment here.
	 */
	if (!IS_ALIGNED(zone_size, BTRFS_STRIPE_LEN)) {
		btrfs_err(fs_info,
			  "zoned: zone size %llu not aligned to stripe %u",
			  zone_size, BTRFS_STRIPE_LEN);
752
		return -EINVAL;
753 754
	}

755 756
	if (btrfs_fs_incompat(fs_info, MIXED_GROUPS)) {
		btrfs_err(fs_info, "zoned: mixed block groups not supported");
757
		return -EINVAL;
758 759
	}

760
	fs_info->zone_size = zone_size;
761 762 763 764 765 766 767 768 769 770 771 772
	/*
	 * Also limit max_zone_append_size by max_segments * PAGE_SIZE.
	 * Technically, we can have multiple pages per segment. But, since
	 * we add the pages one by one to a bio, and cannot increase the
	 * metadata reservation even if it increases the number of extents, it
	 * is safe to stick with the limit.
	 */
	fs_info->max_zone_append_size = ALIGN_DOWN(
		min3((u64)lim->max_zone_append_sectors << SECTOR_SHIFT,
		     (u64)lim->max_sectors << SECTOR_SHIFT,
		     (u64)lim->max_segments << PAGE_SHIFT),
		fs_info->sectorsize);
773
	fs_info->fs_devices->chunk_alloc_policy = BTRFS_CHUNK_ALLOC_ZONED;
774 775
	if (fs_info->max_zone_append_size < fs_info->max_extent_size)
		fs_info->max_extent_size = fs_info->max_zone_append_size;
776

777 778 779 780 781 782
	/*
	 * Check mount options here, because we might change fs_info->zoned
	 * from fs_info->zone_size.
	 */
	ret = btrfs_check_mountopts_zoned(fs_info);
	if (ret)
783
		return ret;
784

785
	btrfs_info(fs_info, "zoned mode enabled with zone size %llu", zone_size);
786
	return 0;
787
}
788 789 790 791 792 793 794 795 796 797 798 799 800 801 802

int btrfs_check_mountopts_zoned(struct btrfs_fs_info *info)
{
	if (!btrfs_is_zoned(info))
		return 0;

	/*
	 * Space cache writing is not COWed. Disable that to avoid write errors
	 * in sequential zones.
	 */
	if (btrfs_test_opt(info, SPACE_CACHE)) {
		btrfs_err(info, "zoned: space cache v1 is not supported");
		return -EINVAL;
	}

803 804 805 806 807
	if (btrfs_test_opt(info, NODATACOW)) {
		btrfs_err(info, "zoned: NODATACOW not supported");
		return -EINVAL;
	}

808 809 810
	btrfs_clear_and_info(info, DISCARD_ASYNC,
			"zoned: async discard ignored and disabled for zoned mode");

811 812
	return 0;
}
813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837

static int sb_log_location(struct block_device *bdev, struct blk_zone *zones,
			   int rw, u64 *bytenr_ret)
{
	u64 wp;
	int ret;

	if (zones[0].type == BLK_ZONE_TYPE_CONVENTIONAL) {
		*bytenr_ret = zones[0].start << SECTOR_SHIFT;
		return 0;
	}

	ret = sb_write_pointer(bdev, zones, &wp);
	if (ret != -ENOENT && ret < 0)
		return ret;

	if (rw == WRITE) {
		struct blk_zone *reset = NULL;

		if (wp == zones[0].start << SECTOR_SHIFT)
			reset = &zones[0];
		else if (wp == zones[1].start << SECTOR_SHIFT)
			reset = &zones[1];

		if (reset && reset->cond != BLK_ZONE_COND_EMPTY) {
838
			ASSERT(sb_zone_is_full(reset));
839 840 841 842 843 844 845 846 847 848 849

			ret = blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET,
					       reset->start, reset->len,
					       GFP_NOFS);
			if (ret)
				return ret;

			reset->cond = BLK_ZONE_COND_EMPTY;
			reset->wp = reset->start;
		}
	} else if (ret != -ENOENT) {
850 851 852 853 854 855
		/*
		 * For READ, we want the previous one. Move write pointer to
		 * the end of a zone, if it is at the head of a zone.
		 */
		u64 zone_end = 0;

856
		if (wp == zones[0].start << SECTOR_SHIFT)
857 858 859 860 861 862 863
			zone_end = zones[1].start + zones[1].capacity;
		else if (wp == zones[1].start << SECTOR_SHIFT)
			zone_end = zones[0].start + zones[0].capacity;
		if (zone_end)
			wp = ALIGN_DOWN(zone_end << SECTOR_SHIFT,
					BTRFS_SUPER_INFO_SIZE);

864 865 866 867 868 869 870 871 872 873 874 875
		wp -= BTRFS_SUPER_INFO_SIZE;
	}

	*bytenr_ret = wp;
	return 0;

}

int btrfs_sb_log_location_bdev(struct block_device *bdev, int mirror, int rw,
			       u64 *bytenr_ret)
{
	struct blk_zone zones[BTRFS_NR_SB_LOG_ZONES];
876
	sector_t zone_sectors;
877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893
	u32 sb_zone;
	int ret;
	u8 zone_sectors_shift;
	sector_t nr_sectors;
	u32 nr_zones;

	if (!bdev_is_zoned(bdev)) {
		*bytenr_ret = btrfs_sb_offset(mirror);
		return 0;
	}

	ASSERT(rw == READ || rw == WRITE);

	zone_sectors = bdev_zone_sectors(bdev);
	if (!is_power_of_2(zone_sectors))
		return -EINVAL;
	zone_sectors_shift = ilog2(zone_sectors);
894
	nr_sectors = bdev_nr_sectors(bdev);
895 896 897 898 899 900
	nr_zones = nr_sectors >> zone_sectors_shift;

	sb_zone = sb_zone_number(zone_sectors_shift + SECTOR_SHIFT, mirror);
	if (sb_zone + 1 >= nr_zones)
		return -ENOENT;

901
	ret = blkdev_report_zones(bdev, zone_start_sector(sb_zone, bdev),
902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917
				  BTRFS_NR_SB_LOG_ZONES, copy_zone_info_cb,
				  zones);
	if (ret < 0)
		return ret;
	if (ret != BTRFS_NR_SB_LOG_ZONES)
		return -EIO;

	return sb_log_location(bdev, zones, rw, bytenr_ret);
}

int btrfs_sb_log_location(struct btrfs_device *device, int mirror, int rw,
			  u64 *bytenr_ret)
{
	struct btrfs_zoned_device_info *zinfo = device->zone_info;
	u32 zone_num;

918 919 920 921 922 923 924
	/*
	 * For a zoned filesystem on a non-zoned block device, use the same
	 * super block locations as regular filesystem. Doing so, the super
	 * block can always be retrieved and the zoned flag of the volume
	 * detected from the super block information.
	 */
	if (!bdev_is_zoned(device->bdev)) {
925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955
		*bytenr_ret = btrfs_sb_offset(mirror);
		return 0;
	}

	zone_num = sb_zone_number(zinfo->zone_size_shift, mirror);
	if (zone_num + 1 >= zinfo->nr_zones)
		return -ENOENT;

	return sb_log_location(device->bdev,
			       &zinfo->sb_zones[BTRFS_NR_SB_LOG_ZONES * mirror],
			       rw, bytenr_ret);
}

static inline bool is_sb_log_zone(struct btrfs_zoned_device_info *zinfo,
				  int mirror)
{
	u32 zone_num;

	if (!zinfo)
		return false;

	zone_num = sb_zone_number(zinfo->zone_size_shift, mirror);
	if (zone_num + 1 >= zinfo->nr_zones)
		return false;

	if (!test_bit(zone_num, zinfo->seq_zones))
		return false;

	return true;
}

956
int btrfs_advance_sb_log(struct btrfs_device *device, int mirror)
957 958 959
{
	struct btrfs_zoned_device_info *zinfo = device->zone_info;
	struct blk_zone *zone;
960
	int i;
961 962

	if (!is_sb_log_zone(zinfo, mirror))
963
		return 0;
964 965

	zone = &zinfo->sb_zones[BTRFS_NR_SB_LOG_ZONES * mirror];
966 967 968 969 970 971 972
	for (i = 0; i < BTRFS_NR_SB_LOG_ZONES; i++) {
		/* Advance the next zone */
		if (zone->cond == BLK_ZONE_COND_FULL) {
			zone++;
			continue;
		}

973 974 975
		if (zone->cond == BLK_ZONE_COND_EMPTY)
			zone->cond = BLK_ZONE_COND_IMP_OPEN;

976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995
		zone->wp += SUPER_INFO_SECTORS;

		if (sb_zone_is_full(zone)) {
			/*
			 * No room left to write new superblock. Since
			 * superblock is written with REQ_SYNC, it is safe to
			 * finish the zone now.
			 *
			 * If the write pointer is exactly at the capacity,
			 * explicit ZONE_FINISH is not necessary.
			 */
			if (zone->wp != zone->start + zone->capacity) {
				int ret;

				ret = blkdev_zone_mgmt(device->bdev,
						REQ_OP_ZONE_FINISH, zone->start,
						zone->len, GFP_NOFS);
				if (ret)
					return ret;
			}
996

997
			zone->wp = zone->start + zone->len;
998
			zone->cond = BLK_ZONE_COND_FULL;
999 1000
		}
		return 0;
1001 1002
	}

1003 1004 1005
	/* All the zones are FULL. Should not reach here. */
	ASSERT(0);
	return -EIO;
1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017
}

int btrfs_reset_sb_log_zones(struct block_device *bdev, int mirror)
{
	sector_t zone_sectors;
	sector_t nr_sectors;
	u8 zone_sectors_shift;
	u32 sb_zone;
	u32 nr_zones;

	zone_sectors = bdev_zone_sectors(bdev);
	zone_sectors_shift = ilog2(zone_sectors);
1018
	nr_sectors = bdev_nr_sectors(bdev);
1019 1020 1021 1022 1023 1024 1025
	nr_zones = nr_sectors >> zone_sectors_shift;

	sb_zone = sb_zone_number(zone_sectors_shift + SECTOR_SHIFT, mirror);
	if (sb_zone + 1 >= nr_zones)
		return -ENOENT;

	return blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET,
1026
				zone_start_sector(sb_zone, bdev),
1027 1028
				zone_sectors * BTRFS_NR_SB_LOG_ZONES, GFP_NOFS);
}
1029

1030 1031
/*
 * Find allocatable zones within a given region.
1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063
 *
 * @device:	the device to allocate a region on
 * @hole_start: the position of the hole to allocate the region
 * @num_bytes:	size of wanted region
 * @hole_end:	the end of the hole
 * @return:	position of allocatable zones
 *
 * Allocatable region should not contain any superblock locations.
 */
u64 btrfs_find_allocatable_zones(struct btrfs_device *device, u64 hole_start,
				 u64 hole_end, u64 num_bytes)
{
	struct btrfs_zoned_device_info *zinfo = device->zone_info;
	const u8 shift = zinfo->zone_size_shift;
	u64 nzones = num_bytes >> shift;
	u64 pos = hole_start;
	u64 begin, end;
	bool have_sb;
	int i;

	ASSERT(IS_ALIGNED(hole_start, zinfo->zone_size));
	ASSERT(IS_ALIGNED(num_bytes, zinfo->zone_size));

	while (pos < hole_end) {
		begin = pos >> shift;
		end = begin + nzones;

		if (end > zinfo->nr_zones)
			return hole_end;

		/* Check if zones in the region are all empty */
		if (btrfs_dev_is_sequential(device, pos) &&
1064
		    !bitmap_test_range_all_set(zinfo->empty_zones, begin, nzones)) {
1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077
			pos += zinfo->zone_size;
			continue;
		}

		have_sb = false;
		for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
			u32 sb_zone;
			u64 sb_pos;

			sb_zone = sb_zone_number(shift, i);
			if (!(end <= sb_zone ||
			      sb_zone + BTRFS_NR_SB_LOG_ZONES <= begin)) {
				have_sb = true;
1078 1079
				pos = zone_start_physical(
					sb_zone + BTRFS_NR_SB_LOG_ZONES, zinfo);
1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099
				break;
			}

			/* We also need to exclude regular superblock positions */
			sb_pos = btrfs_sb_offset(i);
			if (!(pos + num_bytes <= sb_pos ||
			      sb_pos + BTRFS_SUPER_INFO_SIZE <= pos)) {
				have_sb = true;
				pos = ALIGN(sb_pos + BTRFS_SUPER_INFO_SIZE,
					    zinfo->zone_size);
				break;
			}
		}
		if (!have_sb)
			break;
	}

	return pos;
}

1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134
static bool btrfs_dev_set_active_zone(struct btrfs_device *device, u64 pos)
{
	struct btrfs_zoned_device_info *zone_info = device->zone_info;
	unsigned int zno = (pos >> zone_info->zone_size_shift);

	/* We can use any number of zones */
	if (zone_info->max_active_zones == 0)
		return true;

	if (!test_bit(zno, zone_info->active_zones)) {
		/* Active zone left? */
		if (atomic_dec_if_positive(&zone_info->active_zones_left) < 0)
			return false;
		if (test_and_set_bit(zno, zone_info->active_zones)) {
			/* Someone already set the bit */
			atomic_inc(&zone_info->active_zones_left);
		}
	}

	return true;
}

static void btrfs_dev_clear_active_zone(struct btrfs_device *device, u64 pos)
{
	struct btrfs_zoned_device_info *zone_info = device->zone_info;
	unsigned int zno = (pos >> zone_info->zone_size_shift);

	/* We can use any number of zones */
	if (zone_info->max_active_zones == 0)
		return;

	if (test_and_clear_bit(zno, zone_info->active_zones))
		atomic_inc(&zone_info->active_zones_left);
}

1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149
int btrfs_reset_device_zone(struct btrfs_device *device, u64 physical,
			    u64 length, u64 *bytes)
{
	int ret;

	*bytes = 0;
	ret = blkdev_zone_mgmt(device->bdev, REQ_OP_ZONE_RESET,
			       physical >> SECTOR_SHIFT, length >> SECTOR_SHIFT,
			       GFP_NOFS);
	if (ret)
		return ret;

	*bytes = length;
	while (length) {
		btrfs_dev_set_zone_empty(device, physical);
1150
		btrfs_dev_clear_active_zone(device, physical);
1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162
		physical += device->zone_info->zone_size;
		length -= device->zone_info->zone_size;
	}

	return 0;
}

int btrfs_ensure_empty_zones(struct btrfs_device *device, u64 start, u64 size)
{
	struct btrfs_zoned_device_info *zinfo = device->zone_info;
	const u8 shift = zinfo->zone_size_shift;
	unsigned long begin = start >> shift;
1163
	unsigned long nbits = size >> shift;
1164 1165 1166 1167 1168 1169
	u64 pos;
	int ret;

	ASSERT(IS_ALIGNED(start, zinfo->zone_size));
	ASSERT(IS_ALIGNED(size, zinfo->zone_size));

1170
	if (begin + nbits > zinfo->nr_zones)
1171 1172 1173
		return -ERANGE;

	/* All the zones are conventional */
1174
	if (bitmap_test_range_all_zero(zinfo->seq_zones, begin, nbits))
1175 1176 1177
		return 0;

	/* All the zones are sequential and empty */
1178 1179
	if (bitmap_test_range_all_set(zinfo->seq_zones, begin, nbits) &&
	    bitmap_test_range_all_set(zinfo->empty_zones, begin, nbits))
1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203
		return 0;

	for (pos = start; pos < start + size; pos += zinfo->zone_size) {
		u64 reset_bytes;

		if (!btrfs_dev_is_sequential(device, pos) ||
		    btrfs_dev_is_empty_zone(device, pos))
			continue;

		/* Free regions should be empty */
		btrfs_warn_in_rcu(
			device->fs_info,
		"zoned: resetting device %s (devid %llu) zone %llu for allocation",
			rcu_str_deref(device->name), device->devid, pos >> shift);
		WARN_ON_ONCE(1);

		ret = btrfs_reset_device_zone(device, pos, zinfo->zone_size,
					      &reset_bytes);
		if (ret)
			return ret;
	}

	return 0;
}
1204

1205 1206 1207 1208 1209 1210 1211
/*
 * Calculate an allocation pointer from the extent allocation information
 * for a block group consist of conventional zones. It is pointed to the
 * end of the highest addressed extent in the block group as an allocation
 * offset.
 */
static int calculate_alloc_pointer(struct btrfs_block_group *cache,
1212
				   u64 *offset_ret, bool new)
1213 1214
{
	struct btrfs_fs_info *fs_info = cache->fs_info;
1215
	struct btrfs_root *root;
1216 1217 1218 1219 1220 1221
	struct btrfs_path *path;
	struct btrfs_key key;
	struct btrfs_key found_key;
	int ret;
	u64 length;

1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236
	/*
	 * Avoid  tree lookups for a new block group, there's no use for it.
	 * It must always be 0.
	 *
	 * Also, we have a lock chain of extent buffer lock -> chunk mutex.
	 * For new a block group, this function is called from
	 * btrfs_make_block_group() which is already taking the chunk mutex.
	 * Thus, we cannot call calculate_alloc_pointer() which takes extent
	 * buffer locks to avoid deadlock.
	 */
	if (new) {
		*offset_ret = 0;
		return 0;
	}

1237 1238 1239 1240 1241 1242 1243 1244
	path = btrfs_alloc_path();
	if (!path)
		return -ENOMEM;

	key.objectid = cache->start + cache->length;
	key.type = 0;
	key.offset = 0;

1245
	root = btrfs_extent_root(fs_info, key.objectid);
1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282
	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
	/* We should not find the exact match */
	if (!ret)
		ret = -EUCLEAN;
	if (ret < 0)
		goto out;

	ret = btrfs_previous_extent_item(root, path, cache->start);
	if (ret) {
		if (ret == 1) {
			ret = 0;
			*offset_ret = 0;
		}
		goto out;
	}

	btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]);

	if (found_key.type == BTRFS_EXTENT_ITEM_KEY)
		length = found_key.offset;
	else
		length = fs_info->nodesize;

	if (!(found_key.objectid >= cache->start &&
	       found_key.objectid + length <= cache->start + cache->length)) {
		ret = -EUCLEAN;
		goto out;
	}
	*offset_ret = found_key.objectid + length - cache->start;
	ret = 0;

out:
	btrfs_free_path(path);
	return ret;
}

int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache, bool new)
1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294
{
	struct btrfs_fs_info *fs_info = cache->fs_info;
	struct extent_map_tree *em_tree = &fs_info->mapping_tree;
	struct extent_map *em;
	struct map_lookup *map;
	struct btrfs_device *device;
	u64 logical = cache->start;
	u64 length = cache->length;
	int ret;
	int i;
	unsigned int nofs_flag;
	u64 *alloc_offsets = NULL;
1295
	u64 *caps = NULL;
1296
	u64 *physical = NULL;
1297
	unsigned long *active = NULL;
1298
	u64 last_alloc = 0;
1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321
	u32 num_sequential = 0, num_conventional = 0;

	if (!btrfs_is_zoned(fs_info))
		return 0;

	/* Sanity check */
	if (!IS_ALIGNED(length, fs_info->zone_size)) {
		btrfs_err(fs_info,
		"zoned: block group %llu len %llu unaligned to zone size %llu",
			  logical, length, fs_info->zone_size);
		return -EIO;
	}

	/* Get the chunk mapping */
	read_lock(&em_tree->lock);
	em = lookup_extent_mapping(em_tree, logical, length);
	read_unlock(&em_tree->lock);

	if (!em)
		return -EINVAL;

	map = em->map_lookup;

1322
	cache->physical_map = kmemdup(map, map_lookup_size(map->num_stripes), GFP_NOFS);
1323 1324 1325 1326 1327
	if (!cache->physical_map) {
		ret = -ENOMEM;
		goto out;
	}

1328 1329
	alloc_offsets = kcalloc(map->num_stripes, sizeof(*alloc_offsets), GFP_NOFS);
	if (!alloc_offsets) {
1330 1331
		ret = -ENOMEM;
		goto out;
1332 1333
	}

1334 1335 1336 1337 1338 1339
	caps = kcalloc(map->num_stripes, sizeof(*caps), GFP_NOFS);
	if (!caps) {
		ret = -ENOMEM;
		goto out;
	}

1340 1341 1342 1343 1344 1345
	physical = kcalloc(map->num_stripes, sizeof(*physical), GFP_NOFS);
	if (!physical) {
		ret = -ENOMEM;
		goto out;
	}

1346 1347 1348 1349 1350 1351
	active = bitmap_zalloc(map->num_stripes, GFP_NOFS);
	if (!active) {
		ret = -ENOMEM;
		goto out;
	}

1352 1353 1354
	for (i = 0; i < map->num_stripes; i++) {
		bool is_sequential;
		struct blk_zone zone;
1355 1356
		struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
		int dev_replace_is_ongoing = 0;
1357 1358

		device = map->stripes[i].dev;
1359
		physical[i] = map->stripes[i].physical;
1360 1361 1362 1363 1364 1365

		if (device->bdev == NULL) {
			alloc_offsets[i] = WP_MISSING_DEV;
			continue;
		}

1366
		is_sequential = btrfs_dev_is_sequential(device, physical[i]);
1367 1368 1369 1370 1371
		if (is_sequential)
			num_sequential++;
		else
			num_conventional++;

1372 1373 1374 1375 1376 1377 1378
		/*
		 * Consider a zone as active if we can allow any number of
		 * active zones.
		 */
		if (!device->zone_info->max_active_zones)
			__set_bit(i, active);

1379 1380 1381 1382 1383 1384 1385 1386 1387
		if (!is_sequential) {
			alloc_offsets[i] = WP_CONVENTIONAL;
			continue;
		}

		/*
		 * This zone will be used for allocation, so mark this zone
		 * non-empty.
		 */
1388
		btrfs_dev_clear_zone_empty(device, physical[i]);
1389

1390 1391 1392
		down_read(&dev_replace->rwsem);
		dev_replace_is_ongoing = btrfs_dev_replace_is_ongoing(dev_replace);
		if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL)
1393
			btrfs_dev_clear_zone_empty(dev_replace->tgtdev, physical[i]);
1394 1395
		up_read(&dev_replace->rwsem);

1396 1397 1398 1399
		/*
		 * The group is mapped to a sequential zone. Get the zone write
		 * pointer to determine the allocation offset within the zone.
		 */
1400
		WARN_ON(!IS_ALIGNED(physical[i], fs_info->zone_size));
1401
		nofs_flag = memalloc_nofs_save();
1402
		ret = btrfs_get_dev_zone(device, physical[i], &zone);
1403 1404 1405 1406 1407 1408 1409 1410 1411
		memalloc_nofs_restore(nofs_flag);
		if (ret == -EIO || ret == -EOPNOTSUPP) {
			ret = 0;
			alloc_offsets[i] = WP_MISSING_DEV;
			continue;
		} else if (ret) {
			goto out;
		}

1412
		if (zone.type == BLK_ZONE_TYPE_CONVENTIONAL) {
1413 1414 1415 1416
			btrfs_err_in_rcu(fs_info,
	"zoned: unexpected conventional zone %llu on device %s (devid %llu)",
				zone.start << SECTOR_SHIFT,
				rcu_str_deref(device->name), device->devid);
1417 1418 1419 1420
			ret = -EIO;
			goto out;
		}

1421 1422
		caps[i] = (zone.capacity << SECTOR_SHIFT);

1423 1424 1425 1426 1427
		switch (zone.cond) {
		case BLK_ZONE_COND_OFFLINE:
		case BLK_ZONE_COND_READONLY:
			btrfs_err(fs_info,
		"zoned: offline/readonly zone %llu on device %s (devid %llu)",
1428
				  physical[i] >> device->zone_info->zone_size_shift,
1429 1430 1431 1432 1433 1434 1435
				  rcu_str_deref(device->name), device->devid);
			alloc_offsets[i] = WP_MISSING_DEV;
			break;
		case BLK_ZONE_COND_EMPTY:
			alloc_offsets[i] = 0;
			break;
		case BLK_ZONE_COND_FULL:
1436
			alloc_offsets[i] = caps[i];
1437 1438 1439 1440 1441
			break;
		default:
			/* Partially used zone */
			alloc_offsets[i] =
					((zone.wp - zone.start) << SECTOR_SHIFT);
1442
			__set_bit(i, active);
1443 1444 1445 1446
			break;
		}
	}

1447
	if (num_sequential > 0)
1448
		set_bit(BLOCK_GROUP_FLAG_SEQUENTIAL_ZONE, &cache->runtime_flags);
1449

1450
	if (num_conventional > 0) {
1451 1452
		/* Zone capacity is always zone size in emulation */
		cache->zone_capacity = cache->length;
1453 1454 1455
		ret = calculate_alloc_pointer(cache, &last_alloc, new);
		if (ret) {
			btrfs_err(fs_info,
1456
			"zoned: failed to determine allocation offset of bg %llu",
1457 1458 1459 1460
				  cache->start);
			goto out;
		} else if (map->num_stripes == num_conventional) {
			cache->alloc_offset = last_alloc;
1461
			set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &cache->runtime_flags);
1462 1463
			goto out;
		}
1464 1465 1466 1467
	}

	switch (map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) {
	case 0: /* single */
1468 1469 1470
		if (alloc_offsets[0] == WP_MISSING_DEV) {
			btrfs_err(fs_info,
			"zoned: cannot recover write pointer for zone %llu",
1471
				physical[0]);
1472 1473 1474
			ret = -EIO;
			goto out;
		}
1475
		cache->alloc_offset = alloc_offsets[0];
1476
		cache->zone_capacity = caps[0];
1477 1478
		if (test_bit(0, active))
			set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &cache->runtime_flags);
1479 1480
		break;
	case BTRFS_BLOCK_GROUP_DUP:
1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511
		if (map->type & BTRFS_BLOCK_GROUP_DATA) {
			btrfs_err(fs_info, "zoned: profile DUP not yet supported on data bg");
			ret = -EINVAL;
			goto out;
		}
		if (alloc_offsets[0] == WP_MISSING_DEV) {
			btrfs_err(fs_info,
			"zoned: cannot recover write pointer for zone %llu",
				physical[0]);
			ret = -EIO;
			goto out;
		}
		if (alloc_offsets[1] == WP_MISSING_DEV) {
			btrfs_err(fs_info,
			"zoned: cannot recover write pointer for zone %llu",
				physical[1]);
			ret = -EIO;
			goto out;
		}
		if (alloc_offsets[0] != alloc_offsets[1]) {
			btrfs_err(fs_info,
			"zoned: write pointer offset mismatch of zones in DUP profile");
			ret = -EIO;
			goto out;
		}
		if (test_bit(0, active) != test_bit(1, active)) {
			if (!btrfs_zone_activate(cache)) {
				ret = -EIO;
				goto out;
			}
		} else {
1512 1513 1514
			if (test_bit(0, active))
				set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE,
					&cache->runtime_flags);
1515 1516 1517 1518
		}
		cache->alloc_offset = alloc_offsets[0];
		cache->zone_capacity = min(caps[0], caps[1]);
		break;
1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532
	case BTRFS_BLOCK_GROUP_RAID1:
	case BTRFS_BLOCK_GROUP_RAID0:
	case BTRFS_BLOCK_GROUP_RAID10:
	case BTRFS_BLOCK_GROUP_RAID5:
	case BTRFS_BLOCK_GROUP_RAID6:
		/* non-single profiles are not supported yet */
	default:
		btrfs_err(fs_info, "zoned: profile %s not yet supported",
			  btrfs_bg_type_to_raid_name(map->type));
		ret = -EINVAL;
		goto out;
	}

out:
1533 1534 1535 1536 1537 1538 1539
	if (cache->alloc_offset > fs_info->zone_size) {
		btrfs_err(fs_info,
			"zoned: invalid write pointer %llu in block group %llu",
			cache->alloc_offset, cache->start);
		ret = -EIO;
	}

1540 1541 1542 1543 1544 1545 1546 1547
	if (cache->alloc_offset > cache->zone_capacity) {
		btrfs_err(fs_info,
"zoned: invalid write pointer %llu (larger than zone capacity %llu) in block group %llu",
			  cache->alloc_offset, cache->zone_capacity,
			  cache->start);
		ret = -EIO;
	}

1548 1549 1550 1551 1552 1553 1554 1555
	/* An extent is allocated after the write pointer */
	if (!ret && num_conventional && last_alloc > cache->alloc_offset) {
		btrfs_err(fs_info,
			  "zoned: got wrong write pointer in BG %llu: %llu > %llu",
			  logical, last_alloc, cache->alloc_offset);
		ret = -EIO;
	}

1556
	if (!ret) {
1557
		cache->meta_write_pointer = cache->alloc_offset + cache->start;
1558
		if (test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &cache->runtime_flags)) {
1559 1560 1561 1562 1563 1564 1565
			btrfs_get_block_group(cache);
			spin_lock(&fs_info->zone_active_bgs_lock);
			list_add_tail(&cache->active_bg_list,
				      &fs_info->zone_active_bgs);
			spin_unlock(&fs_info->zone_active_bgs_lock);
		}
	} else {
1566 1567 1568
		kfree(cache->physical_map);
		cache->physical_map = NULL;
	}
1569
	bitmap_free(active);
1570
	kfree(physical);
1571
	kfree(caps);
1572 1573 1574 1575 1576
	kfree(alloc_offsets);
	free_extent_map(em);

	return ret;
}
1577 1578 1579 1580 1581 1582 1583 1584 1585

void btrfs_calc_zone_unusable(struct btrfs_block_group *cache)
{
	u64 unusable, free;

	if (!btrfs_is_zoned(cache->fs_info))
		return;

	WARN_ON(cache->bytes_super != 0);
1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598

	/* Check for block groups never get activated */
	if (test_bit(BTRFS_FS_ACTIVE_ZONE_TRACKING, &cache->fs_info->flags) &&
	    cache->flags & (BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_SYSTEM) &&
	    !test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &cache->runtime_flags) &&
	    cache->alloc_offset == 0) {
		unusable = cache->length;
		free = 0;
	} else {
		unusable = (cache->alloc_offset - cache->used) +
			   (cache->length - cache->zone_capacity);
		free = cache->zone_capacity - cache->alloc_offset;
	}
1599 1600 1601 1602 1603 1604

	/* We only need ->free_space in ALLOC_SEQ block groups */
	cache->cached = BTRFS_CACHE_FINISHED;
	cache->free_space_ctl->free_space = free;
	cache->zone_unusable = unusable;
}
1605 1606 1607 1608

void btrfs_redirty_list_add(struct btrfs_transaction *trans,
			    struct extent_buffer *eb)
{
1609 1610
	if (!btrfs_is_zoned(eb->fs_info) ||
	    btrfs_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN))
1611 1612
		return;

1613 1614
	ASSERT(!test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));

1615 1616
	memzero_extent_buffer(eb, 0, eb->len);
	set_bit(EXTENT_BUFFER_NO_CHECK, &eb->bflags);
1617
	set_extent_buffer_dirty(eb);
1618
	set_extent_bit(&trans->dirty_pages, eb->start, eb->start + eb->len - 1,
1619
			EXTENT_DIRTY | EXTENT_NOWAIT, NULL);
1620
}
1621

1622
bool btrfs_use_zone_append(struct btrfs_bio *bbio)
1623
{
1624 1625
	u64 start = (bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT);
	struct btrfs_inode *inode = bbio->inode;
1626
	struct btrfs_fs_info *fs_info = bbio->fs_info;
1627 1628 1629 1630 1631 1632
	struct btrfs_block_group *cache;
	bool ret = false;

	if (!btrfs_is_zoned(fs_info))
		return false;

1633
	if (!inode || !is_data_inode(&inode->vfs_inode))
1634 1635
		return false;

1636 1637 1638
	if (btrfs_op(&bbio->bio) != BTRFS_MAP_WRITE)
		return false;

1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649
	/*
	 * Using REQ_OP_ZONE_APPNED for relocation can break assumptions on the
	 * extent layout the relocation code has.
	 * Furthermore we have set aside own block-group from which only the
	 * relocation "process" can allocate and make sure only one process at a
	 * time can add pages to an extent that gets relocated, so it's safe to
	 * use regular REQ_OP_WRITE for this special case.
	 */
	if (btrfs_is_data_reloc_root(inode->root))
		return false;

1650
	cache = btrfs_lookup_block_group(fs_info, start);
1651 1652 1653 1654
	ASSERT(cache);
	if (!cache)
		return false;

1655
	ret = !!test_bit(BLOCK_GROUP_FLAG_SEQUENTIAL_ZONE, &cache->runtime_flags);
1656 1657 1658 1659
	btrfs_put_block_group(cache);

	return ret;
}
1660

1661
void btrfs_record_physical_zoned(struct btrfs_bio *bbio)
1662
{
1663
	const u64 physical = bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT;
1664
	struct btrfs_ordered_sum *sum = bbio->sums;
1665

1666 1667 1668 1669
	if (physical < bbio->orig_physical)
		sum->logical -= bbio->orig_physical - physical;
	else
		sum->logical += physical - bbio->orig_physical;
1670 1671
}

1672 1673
static void btrfs_rewrite_logical_zoned(struct btrfs_ordered_extent *ordered,
					u64 logical)
1674
{
1675
	struct extent_map_tree *em_tree = &BTRFS_I(ordered->inode)->extent_tree;
1676 1677
	struct extent_map *em;

1678
	ordered->disk_bytenr = logical;
1679 1680 1681 1682

	write_lock(&em_tree->lock);
	em = search_extent_mapping(em_tree, ordered->file_offset,
				   ordered->num_bytes);
1683
	em->block_start = logical;
1684 1685
	free_extent_map(em);
	write_unlock(&em_tree->lock);
1686 1687 1688 1689 1690 1691 1692 1693 1694
}

static bool btrfs_zoned_split_ordered(struct btrfs_ordered_extent *ordered,
				      u64 logical, u64 len)
{
	struct btrfs_ordered_extent *new;

	if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered->flags) &&
	    split_extent_map(BTRFS_I(ordered->inode), ordered->file_offset,
1695
			     ordered->num_bytes, len, logical))
1696 1697 1698 1699 1700
		return false;

	new = btrfs_split_ordered_extent(ordered, len);
	if (IS_ERR(new))
		return false;
1701
	new->disk_bytenr = logical;
1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731
	btrfs_finish_one_ordered(new);
	return true;
}

void btrfs_finish_ordered_zoned(struct btrfs_ordered_extent *ordered)
{
	struct btrfs_inode *inode = BTRFS_I(ordered->inode);
	struct btrfs_fs_info *fs_info = inode->root->fs_info;
	struct btrfs_ordered_sum *sum =
		list_first_entry(&ordered->list, typeof(*sum), list);
	u64 logical = sum->logical;
	u64 len = sum->len;

	while (len < ordered->disk_num_bytes) {
		sum = list_next_entry(sum, list);
		if (sum->logical == logical + len) {
			len += sum->len;
			continue;
		}
		if (!btrfs_zoned_split_ordered(ordered, logical, len)) {
			set_bit(BTRFS_ORDERED_IOERR, &ordered->flags);
			btrfs_err(fs_info, "failed to split ordered extent");
			goto out;
		}
		logical = sum->logical;
		len = sum->len;
	}

	if (ordered->disk_bytenr != logical)
		btrfs_rewrite_logical_zoned(ordered, logical);
1732

1733 1734 1735 1736 1737 1738 1739 1740
out:
	/*
	 * If we end up here for nodatasum I/O, the btrfs_ordered_sum structures
	 * were allocated by btrfs_alloc_dummy_sum only to record the logical
	 * addresses and don't contain actual checksums.  We thus must free them
	 * here so that we don't attempt to log the csums later.
	 */
	if ((inode->flags & BTRFS_INODE_NODATASUM) ||
1741 1742 1743 1744 1745 1746
	    test_bit(BTRFS_FS_STATE_NO_CSUMS, &fs_info->fs_state)) {
		while ((sum = list_first_entry_or_null(&ordered->list,
						       typeof(*sum), list))) {
			list_del(&sum->list);
			kfree(sum);
		}
1747 1748
	}
}
1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759

bool btrfs_check_meta_write_pointer(struct btrfs_fs_info *fs_info,
				    struct extent_buffer *eb,
				    struct btrfs_block_group **cache_ret)
{
	struct btrfs_block_group *cache;
	bool ret = true;

	if (!btrfs_is_zoned(fs_info))
		return true;

1760 1761 1762
	cache = btrfs_lookup_block_group(fs_info, eb->start);
	if (!cache)
		return true;
1763

1764
	if (cache->meta_write_pointer != eb->start) {
1765 1766
		btrfs_put_block_group(cache);
		cache = NULL;
1767 1768 1769
		ret = false;
	} else {
		cache->meta_write_pointer = eb->start + eb->len;
1770 1771
	}

1772
	*cache_ret = cache;
1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785

	return ret;
}

void btrfs_revert_meta_write_pointer(struct btrfs_block_group *cache,
				     struct extent_buffer *eb)
{
	if (!btrfs_is_zoned(eb->fs_info) || !cache)
		return;

	ASSERT(cache->meta_write_pointer == eb->start + eb->len);
	cache->meta_write_pointer = eb->start;
}
1786 1787 1788 1789 1790 1791 1792 1793 1794

int btrfs_zoned_issue_zeroout(struct btrfs_device *device, u64 physical, u64 length)
{
	if (!btrfs_dev_is_sequential(device, physical))
		return -EOPNOTSUPP;

	return blkdev_issue_zeroout(device->bdev, physical >> SECTOR_SHIFT,
				    length >> SECTOR_SHIFT, GFP_NOFS, 0);
}
1795 1796 1797 1798

static int read_zone_info(struct btrfs_fs_info *fs_info, u64 logical,
			  struct blk_zone *zone)
{
1799
	struct btrfs_io_context *bioc = NULL;
1800 1801 1802 1803 1804
	u64 mapped_length = PAGE_SIZE;
	unsigned int nofs_flag;
	int nmirrors;
	int i, ret;

1805 1806
	ret = btrfs_map_block(fs_info, BTRFS_MAP_GET_READ_MIRRORS, logical,
			      &mapped_length, &bioc, NULL, NULL, 1);
1807
	if (ret || !bioc || mapped_length < PAGE_SIZE) {
1808 1809
		ret = -EIO;
		goto out_put_bioc;
1810 1811
	}

1812 1813 1814 1815
	if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
		ret = -EINVAL;
		goto out_put_bioc;
	}
1816 1817

	nofs_flag = memalloc_nofs_save();
1818
	nmirrors = (int)bioc->num_stripes;
1819
	for (i = 0; i < nmirrors; i++) {
1820 1821
		u64 physical = bioc->stripes[i].physical;
		struct btrfs_device *dev = bioc->stripes[i].dev;
1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833

		/* Missing device */
		if (!dev->bdev)
			continue;

		ret = btrfs_get_dev_zone(dev, physical, zone);
		/* Failing device */
		if (ret == -EIO || ret == -EOPNOTSUPP)
			continue;
		break;
	}
	memalloc_nofs_restore(nofs_flag);
1834 1835
out_put_bioc:
	btrfs_put_bioc(bioc);
1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870
	return ret;
}

/*
 * Synchronize write pointer in a zone at @physical_start on @tgt_dev, by
 * filling zeros between @physical_pos to a write pointer of dev-replace
 * source device.
 */
int btrfs_sync_zone_write_pointer(struct btrfs_device *tgt_dev, u64 logical,
				    u64 physical_start, u64 physical_pos)
{
	struct btrfs_fs_info *fs_info = tgt_dev->fs_info;
	struct blk_zone zone;
	u64 length;
	u64 wp;
	int ret;

	if (!btrfs_dev_is_sequential(tgt_dev, physical_pos))
		return 0;

	ret = read_zone_info(fs_info, logical, &zone);
	if (ret)
		return ret;

	wp = physical_start + ((zone.wp - zone.start) << SECTOR_SHIFT);

	if (physical_pos == wp)
		return 0;

	if (physical_pos > wp)
		return -EUCLEAN;

	length = wp - physical_pos;
	return btrfs_zoned_issue_zeroout(tgt_dev, physical_pos, length);
}
1871

1872
/*
1873 1874 1875 1876 1877 1878 1879 1880 1881
 * Activate block group and underlying device zones
 *
 * @block_group: the block group to activate
 *
 * Return: true on success, false otherwise
 */
bool btrfs_zone_activate(struct btrfs_block_group *block_group)
{
	struct btrfs_fs_info *fs_info = block_group->fs_info;
1882
	struct btrfs_space_info *space_info = block_group->space_info;
1883 1884 1885 1886
	struct map_lookup *map;
	struct btrfs_device *device;
	u64 physical;
	bool ret;
1887
	int i;
1888 1889 1890 1891 1892 1893

	if (!btrfs_is_zoned(block_group->fs_info))
		return true;

	map = block_group->physical_map;

1894
	spin_lock(&space_info->lock);
1895
	spin_lock(&block_group->lock);
1896
	if (test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &block_group->runtime_flags)) {
1897 1898 1899 1900
		ret = true;
		goto out_unlock;
	}

1901
	/* No space left */
1902
	if (btrfs_zoned_bg_is_full(block_group)) {
1903 1904 1905 1906
		ret = false;
		goto out_unlock;
	}

1907 1908 1909
	for (i = 0; i < map->num_stripes; i++) {
		device = map->stripes[i].dev;
		physical = map->stripes[i].physical;
1910

1911 1912 1913 1914 1915 1916 1917 1918 1919
		if (device->zone_info->max_active_zones == 0)
			continue;

		if (!btrfs_dev_set_active_zone(device, physical)) {
			/* Cannot activate the zone */
			ret = false;
			goto out_unlock;
		}
	}
1920 1921

	/* Successfully activated all the zones */
1922
	set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &block_group->runtime_flags);
1923 1924 1925 1926 1927
	WARN_ON(block_group->alloc_offset != 0);
	if (block_group->zone_unusable == block_group->length) {
		block_group->zone_unusable = block_group->length - block_group->zone_capacity;
		space_info->bytes_zone_unusable -= block_group->zone_capacity;
	}
1928
	spin_unlock(&block_group->lock);
1929 1930
	btrfs_try_granting_tickets(fs_info, space_info);
	spin_unlock(&space_info->lock);
1931

1932 1933
	/* For the active block group list */
	btrfs_get_block_group(block_group);
1934

1935 1936 1937
	spin_lock(&fs_info->zone_active_bgs_lock);
	list_add_tail(&block_group->active_bg_list, &fs_info->zone_active_bgs);
	spin_unlock(&fs_info->zone_active_bgs_lock);
1938 1939 1940 1941 1942

	return true;

out_unlock:
	spin_unlock(&block_group->lock);
1943
	spin_unlock(&space_info->lock);
1944 1945 1946
	return ret;
}

1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978
static void wait_eb_writebacks(struct btrfs_block_group *block_group)
{
	struct btrfs_fs_info *fs_info = block_group->fs_info;
	const u64 end = block_group->start + block_group->length;
	struct radix_tree_iter iter;
	struct extent_buffer *eb;
	void __rcu **slot;

	rcu_read_lock();
	radix_tree_for_each_slot(slot, &fs_info->buffer_radix, &iter,
				 block_group->start >> fs_info->sectorsize_bits) {
		eb = radix_tree_deref_slot(slot);
		if (!eb)
			continue;
		if (radix_tree_deref_retry(eb)) {
			slot = radix_tree_iter_retry(&iter);
			continue;
		}

		if (eb->start < block_group->start)
			continue;
		if (eb->start >= end)
			break;

		slot = radix_tree_iter_resume(slot, &iter);
		rcu_read_unlock();
		wait_on_extent_buffer_writeback(eb);
		rcu_read_lock();
	}
	rcu_read_unlock();
}

1979
static int do_zone_finish(struct btrfs_block_group *block_group, bool fully_written)
1980 1981 1982
{
	struct btrfs_fs_info *fs_info = block_group->fs_info;
	struct map_lookup *map;
1983 1984
	const bool is_metadata = (block_group->flags &
			(BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_SYSTEM));
1985
	int ret = 0;
1986
	int i;
1987 1988

	spin_lock(&block_group->lock);
1989
	if (!test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &block_group->runtime_flags)) {
1990 1991 1992 1993 1994
		spin_unlock(&block_group->lock);
		return 0;
	}

	/* Check if we have unwritten allocated space */
1995
	if (is_metadata &&
1996
	    block_group->start + block_group->alloc_offset > block_group->meta_write_pointer) {
1997 1998 1999 2000 2001
		spin_unlock(&block_group->lock);
		return -EAGAIN;
	}

	/*
2002 2003 2004 2005 2006
	 * If we are sure that the block group is full (= no more room left for
	 * new allocation) and the IO for the last usable block is completed, we
	 * don't need to wait for the other IOs. This holds because we ensure
	 * the sequential IO submissions using the ZONE_APPEND command for data
	 * and block_group->meta_write_pointer for metadata.
2007
	 */
2008
	if (!fully_written) {
2009 2010
		spin_unlock(&block_group->lock);

2011 2012 2013 2014 2015 2016 2017 2018 2019
		ret = btrfs_inc_block_group_ro(block_group, false);
		if (ret)
			return ret;

		/* Ensure all writes in this block group finish */
		btrfs_wait_block_group_reservations(block_group);
		/* No need to wait for NOCOW writers. Zoned mode does not allow that */
		btrfs_wait_ordered_roots(fs_info, U64_MAX, block_group->start,
					 block_group->length);
2020 2021 2022
		/* Wait for extent buffers to be written. */
		if (is_metadata)
			wait_eb_writebacks(block_group);
2023 2024 2025 2026 2027 2028 2029

		spin_lock(&block_group->lock);

		/*
		 * Bail out if someone already deactivated the block group, or
		 * allocated space is left in the block group.
		 */
2030 2031
		if (!test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE,
			      &block_group->runtime_flags)) {
2032 2033 2034 2035 2036 2037 2038 2039 2040 2041
			spin_unlock(&block_group->lock);
			btrfs_dec_block_group_ro(block_group);
			return 0;
		}

		if (block_group->reserved) {
			spin_unlock(&block_group->lock);
			btrfs_dec_block_group_ro(block_group);
			return -EAGAIN;
		}
2042 2043
	}

2044
	clear_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &block_group->runtime_flags);
2045 2046 2047
	block_group->alloc_offset = block_group->zone_capacity;
	block_group->free_space_ctl->free_space = 0;
	btrfs_clear_treelog_bg(block_group);
2048
	btrfs_clear_data_reloc_bg(block_group);
2049 2050
	spin_unlock(&block_group->lock);

2051
	map = block_group->physical_map;
2052
	for (i = 0; i < map->num_stripes; i++) {
2053 2054
		struct btrfs_device *device = map->stripes[i].dev;
		const u64 physical = map->stripes[i].physical;
2055

2056 2057
		if (device->zone_info->max_active_zones == 0)
			continue;
2058

2059 2060 2061 2062
		ret = blkdev_zone_mgmt(device->bdev, REQ_OP_ZONE_FINISH,
				       physical >> SECTOR_SHIFT,
				       device->zone_info->zone_size >> SECTOR_SHIFT,
				       GFP_NOFS);
2063

2064 2065
		if (ret)
			return ret;
2066

2067
		btrfs_dev_clear_active_zone(device, physical);
2068
	}
2069 2070 2071

	if (!fully_written)
		btrfs_dec_block_group_ro(block_group);
2072

2073 2074 2075 2076 2077 2078 2079 2080
	spin_lock(&fs_info->zone_active_bgs_lock);
	ASSERT(!list_empty(&block_group->active_bg_list));
	list_del_init(&block_group->active_bg_list);
	spin_unlock(&fs_info->zone_active_bgs_lock);

	/* For active_bg_list */
	btrfs_put_block_group(block_group);

2081
	clear_and_wake_up_bit(BTRFS_FS_NEED_ZONE_FINISH, &fs_info->flags);
2082

2083
	return 0;
2084
}
2085

2086 2087 2088 2089 2090 2091 2092 2093
int btrfs_zone_finish(struct btrfs_block_group *block_group)
{
	if (!btrfs_is_zoned(block_group->fs_info))
		return 0;

	return do_zone_finish(block_group, false);
}

2094
bool btrfs_can_activate_zone(struct btrfs_fs_devices *fs_devices, u64 flags)
2095
{
2096
	struct btrfs_fs_info *fs_info = fs_devices->fs_info;
2097 2098 2099
	struct btrfs_device *device;
	bool ret = false;

2100
	if (!btrfs_is_zoned(fs_info))
2101 2102 2103
		return true;

	/* Check if there is a device with active zones left */
2104 2105
	mutex_lock(&fs_info->chunk_mutex);
	list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) {
2106 2107 2108 2109 2110
		struct btrfs_zoned_device_info *zinfo = device->zone_info;

		if (!device->bdev)
			continue;

2111
		if (!zinfo->max_active_zones) {
2112 2113 2114
			ret = true;
			break;
		}
2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125

		switch (flags & BTRFS_BLOCK_GROUP_PROFILE_MASK) {
		case 0: /* single */
			ret = (atomic_read(&zinfo->active_zones_left) >= 1);
			break;
		case BTRFS_BLOCK_GROUP_DUP:
			ret = (atomic_read(&zinfo->active_zones_left) >= 2);
			break;
		}
		if (ret)
			break;
2126
	}
2127
	mutex_unlock(&fs_info->chunk_mutex);
2128

2129 2130 2131
	if (!ret)
		set_bit(BTRFS_FS_NEED_ZONE_FINISH, &fs_info->flags);

2132 2133
	return ret;
}
2134 2135 2136 2137

void btrfs_zone_finish_endio(struct btrfs_fs_info *fs_info, u64 logical, u64 length)
{
	struct btrfs_block_group *block_group;
2138
	u64 min_alloc_bytes;
2139 2140 2141 2142 2143 2144 2145

	if (!btrfs_is_zoned(fs_info))
		return;

	block_group = btrfs_lookup_block_group(fs_info, logical);
	ASSERT(block_group);

2146 2147 2148 2149 2150
	/* No MIXED_BG on zoned btrfs. */
	if (block_group->flags & BTRFS_BLOCK_GROUP_DATA)
		min_alloc_bytes = fs_info->sectorsize;
	else
		min_alloc_bytes = fs_info->nodesize;
2151

2152 2153 2154
	/* Bail out if we can allocate more data from this block group. */
	if (logical + length + min_alloc_bytes <=
	    block_group->start + block_group->zone_capacity)
2155 2156
		goto out;

2157
	do_zone_finish(block_group, true);
2158 2159 2160 2161 2162

out:
	btrfs_put_block_group(block_group);
}

2163 2164 2165 2166
static void btrfs_zone_finish_endio_workfn(struct work_struct *work)
{
	struct btrfs_block_group *bg =
		container_of(work, struct btrfs_block_group, zone_finish_work);
2167

2168 2169 2170 2171 2172
	wait_on_extent_buffer_writeback(bg->last_eb);
	free_extent_buffer(bg->last_eb);
	btrfs_zone_finish_endio(bg->fs_info, bg->start, bg->length);
	btrfs_put_block_group(bg);
}
2173

2174 2175 2176
void btrfs_schedule_zone_finish_bg(struct btrfs_block_group *bg,
				   struct extent_buffer *eb)
{
2177 2178
	if (!test_bit(BLOCK_GROUP_FLAG_SEQUENTIAL_ZONE, &bg->runtime_flags) ||
	    eb->start + eb->len * 2 <= bg->start + bg->zone_capacity)
2179
		return;
2180

2181 2182 2183 2184 2185
	if (WARN_ON(bg->zone_finish_work.func == btrfs_zone_finish_endio_workfn)) {
		btrfs_err(bg->fs_info, "double scheduling of bg %llu zone finishing",
			  bg->start);
		return;
	}
2186

2187 2188 2189 2190 2191 2192
	/* For the work */
	btrfs_get_block_group(bg);
	atomic_inc(&eb->refs);
	bg->last_eb = eb;
	INIT_WORK(&bg->zone_finish_work, btrfs_zone_finish_endio_workfn);
	queue_work(system_unbound_wq, &bg->zone_finish_work);
2193
}
2194 2195 2196 2197 2198 2199 2200 2201 2202 2203

void btrfs_clear_data_reloc_bg(struct btrfs_block_group *bg)
{
	struct btrfs_fs_info *fs_info = bg->fs_info;

	spin_lock(&fs_info->relocation_bg_lock);
	if (fs_info->data_reloc_bg == bg->start)
		fs_info->data_reloc_bg = 0;
	spin_unlock(&fs_info->relocation_bg_lock);
}
2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221

void btrfs_free_zone_cache(struct btrfs_fs_info *fs_info)
{
	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
	struct btrfs_device *device;

	if (!btrfs_is_zoned(fs_info))
		return;

	mutex_lock(&fs_devices->device_list_mutex);
	list_for_each_entry(device, &fs_devices->devices, dev_list) {
		if (device->zone_info) {
			vfree(device->zone_info->zone_cache);
			device->zone_info->zone_cache = NULL;
		}
	}
	mutex_unlock(&fs_devices->device_list_mutex);
}
2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248

bool btrfs_zoned_should_reclaim(struct btrfs_fs_info *fs_info)
{
	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
	struct btrfs_device *device;
	u64 used = 0;
	u64 total = 0;
	u64 factor;

	ASSERT(btrfs_is_zoned(fs_info));

	if (fs_info->bg_reclaim_threshold == 0)
		return false;

	mutex_lock(&fs_devices->device_list_mutex);
	list_for_each_entry(device, &fs_devices->devices, dev_list) {
		if (!device->bdev)
			continue;

		total += device->disk_total_bytes;
		used += device->bytes_used;
	}
	mutex_unlock(&fs_devices->device_list_mutex);

	factor = div64_u64(used * 100, total);
	return factor >= fs_info->bg_reclaim_threshold;
}
2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262

void btrfs_zoned_release_data_reloc_bg(struct btrfs_fs_info *fs_info, u64 logical,
				       u64 length)
{
	struct btrfs_block_group *block_group;

	if (!btrfs_is_zoned(fs_info))
		return;

	block_group = btrfs_lookup_block_group(fs_info, logical);
	/* It should be called on a previous data relocation block group. */
	ASSERT(block_group && (block_group->flags & BTRFS_BLOCK_GROUP_DATA));

	spin_lock(&block_group->lock);
2263
	if (!test_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, &block_group->runtime_flags))
2264 2265 2266 2267 2268
		goto out;

	/* All relocation extents are written. */
	if (block_group->start + block_group->alloc_offset == logical + length) {
		/* Now, release this block group for further allocations. */
2269 2270
		clear_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC,
			  &block_group->runtime_flags);
2271 2272 2273 2274 2275 2276
	}

out:
	spin_unlock(&block_group->lock);
	btrfs_put_block_group(block_group);
}
2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290

int btrfs_zone_finish_one_bg(struct btrfs_fs_info *fs_info)
{
	struct btrfs_block_group *block_group;
	struct btrfs_block_group *min_bg = NULL;
	u64 min_avail = U64_MAX;
	int ret;

	spin_lock(&fs_info->zone_active_bgs_lock);
	list_for_each_entry(block_group, &fs_info->zone_active_bgs,
			    active_bg_list) {
		u64 avail;

		spin_lock(&block_group->lock);
2291
		if (block_group->reserved || block_group->alloc_offset == 0 ||
2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316
		    (block_group->flags & BTRFS_BLOCK_GROUP_SYSTEM)) {
			spin_unlock(&block_group->lock);
			continue;
		}

		avail = block_group->zone_capacity - block_group->alloc_offset;
		if (min_avail > avail) {
			if (min_bg)
				btrfs_put_block_group(min_bg);
			min_bg = block_group;
			min_avail = avail;
			btrfs_get_block_group(min_bg);
		}
		spin_unlock(&block_group->lock);
	}
	spin_unlock(&fs_info->zone_active_bgs_lock);

	if (!min_bg)
		return 0;

	ret = btrfs_zone_finish(min_bg);
	btrfs_put_block_group(min_bg);

	return ret < 0 ? ret : 1;
}
2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337

int btrfs_zoned_activate_one_bg(struct btrfs_fs_info *fs_info,
				struct btrfs_space_info *space_info,
				bool do_finish)
{
	struct btrfs_block_group *bg;
	int index;

	if (!btrfs_is_zoned(fs_info) || (space_info->flags & BTRFS_BLOCK_GROUP_DATA))
		return 0;

	for (;;) {
		int ret;
		bool need_finish = false;

		down_read(&space_info->groups_sem);
		for (index = 0; index < BTRFS_NR_RAID_TYPES; index++) {
			list_for_each_entry(bg, &space_info->block_groups[index],
					    list) {
				if (!spin_trylock(&bg->lock))
					continue;
2338 2339 2340
				if (btrfs_zoned_bg_is_full(bg) ||
				    test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE,
					     &bg->runtime_flags)) {
2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367
					spin_unlock(&bg->lock);
					continue;
				}
				spin_unlock(&bg->lock);

				if (btrfs_zone_activate(bg)) {
					up_read(&space_info->groups_sem);
					return 1;
				}

				need_finish = true;
			}
		}
		up_read(&space_info->groups_sem);

		if (!do_finish || !need_finish)
			break;

		ret = btrfs_zone_finish_one_bg(fs_info);
		if (ret == 0)
			break;
		if (ret < 0)
			return ret;
	}

	return 0;
}