bcachefs: swap file support via SWP_FS_OPS

fs/Makefile

@@ -189,7 +189,8 @@ bcachefs-y		:=			\
 	vfs/io.o				\
 	vfs/buffered.o				\
 	vfs/direct.o				\
-	vfs/pagecache.o
+	vfs/pagecache.o				\
+	vfs/swap.o
 
 ifdef CONFIG_DEBUG_FS
 	bcachefs-y += debug/async_objs.o

fs/btree/bkey_buf.h

@@ -56,6 +56,12 @@ static inline void bch2_bkey_buf_init(struct bkey_buf *s)
 	bkey_init(&s->k->k);
 }
 
+static inline void bch2_bkey_buf_init_prealloc(struct bkey_buf *s, void *buf)
+{
+	s->k = buf;
+	bkey_init(&s->k->k);
+}
+
 static inline void bch2_bkey_buf_exit(struct bkey_buf *s)
 {
 	if (s->k != (void *) s->onstack)

fs/btree/cache.c

@@ -137,6 +137,67 @@ void bch2_recalc_btree_reserve(struct bch_fs *c)
 	c->btree.cache.nr_reserve = reserve;
 }
 
+/*
+ * bch2_btree_cache_add_reserve - pre-allocate btree node buffers for swap I/O
+ *
+ * At swapon time, allocate @count empty btree node buffers and put them on
+ * bc->freeable.  bch2_btree_node_mem_alloc() always checks freeable first and
+ * steals the data buffer from a freeable node rather than going to the page
+ * allocator — so under PF_MEMALLOC these buffers are used without hitting the
+ * emergency reserve at all.
+ *
+ * A single swap COW write touches at least three btrees (extents, inodes, and
+ * alloc/freespace), each up to BTREE_MAX_DEPTH levels deep.  With mempool=8
+ * the worst case is ~24 MB of concurrent node buffers; callers should reserve
+ * enough to cover their expected concurrency.
+ *
+ * We also increment nr_reserve by the same count.  nr_reserve is the shrinker
+ * eviction watermark for bc->live[0]; raising it reduces the shrinker's
+ * can_free budget, which indirectly shields the freeable pool from being
+ * drained under memory pressure before swap I/O can use it.
+ *
+ * Returns the number of nodes actually allocated (may be less than @count on
+ * ENOMEM).
+ */
+static struct btree *__bch2_btree_node_mem_alloc_gfp(struct bch_fs *, gfp_t);
+
+int bch2_btree_cache_add_reserve(struct bch_fs *c, unsigned count)
+{
+	struct bch_fs_btree_cache *bc = &c->btree.cache;
+	unsigned allocated = 0;
+
+	for (unsigned i = 0; i < count; i++) {
+		struct btree *b = __bch2_btree_node_mem_alloc_gfp(c,
+					GFP_KERNEL | __GFP_NORETRY);
+
+		if (!b)
+			break;
+
+		scoped_guard(mutex, &bc->lock) {
+			bch2_btree_node_transition_state_locked(bc, b, BTREE_NODE_CACHE_FREEABLE);
+			bc->nr_reserve++;
+		}
+		allocated++;
+	}
+
+	return allocated;
+}
+
+/*
+ * bch2_btree_cache_remove_reserve - release swap-time btree node reserve
+ *
+ * Called at swapoff.  Decrements nr_reserve so the shrinker can reclaim the
+ * extra freeable buffers under normal memory pressure.  The actual kvfree of
+ * the node data buffers happens lazily via the shrinker scan.
+ */
+void bch2_btree_cache_remove_reserve(struct bch_fs *c, unsigned count)
+{
+	struct bch_fs_btree_cache *bc = &c->btree.cache;
+
+	scoped_guard(mutex, &bc->lock)
+		bc->nr_reserve -= min(bc->nr_reserve, (size_t)count);
+}
+
 /* Btree node allocation */
 
 struct btree_node_bufs {
@@ -280,13 +341,13 @@ static struct btree *__btree_node_mem_alloc(struct bch_fs *c, bool pcpu_read_loc
 	return b;
 }
 
-struct btree *__bch2_btree_node_mem_alloc(struct bch_fs *c)
+static struct btree *__bch2_btree_node_mem_alloc_gfp(struct bch_fs *c, gfp_t gfp)
 {
 	struct btree_node_bufs bufs = { .byte_order = ilog2(c->opts.btree_node_size) };
 	struct btree *b;
 
-	if (__btree_node_data_alloc(c, &bufs, GFP_KERNEL, false) ||
-	    !(b = __btree_node_mem_alloc(c, false, GFP_KERNEL))) {
+	if (__btree_node_data_alloc(c, &bufs, gfp, false) ||
+	    !(b = __btree_node_mem_alloc(c, false, gfp))) {
 		btree_node_bufs_free(&bufs);
 		return NULL;
 	}
@@ -296,6 +357,11 @@ struct btree *__bch2_btree_node_mem_alloc(struct bch_fs *c)
 	return b;
 }
 
+struct btree *__bch2_btree_node_mem_alloc(struct bch_fs *c)
+{
+	return __bch2_btree_node_mem_alloc_gfp(c, GFP_KERNEL);
+}
+
 /* Pinning */
 
 static inline bool __btree_node_pinned(struct bch_fs_btree_cache *bc, struct btree *b)

fs/btree/cache.h

@@ -11,6 +11,8 @@ extern const char * const bch2_btree_node_flags[];
 struct btree_iter;
 
 void bch2_recalc_btree_reserve(struct bch_fs *);
+int  bch2_btree_cache_add_reserve(struct bch_fs *, unsigned count);
+void bch2_btree_cache_remove_reserve(struct bch_fs *, unsigned count);
 
 void bch2_btree_node_mem_free(struct bch_fs *, struct btree *);
 

fs/btree/iter.c

@@ -2001,7 +2001,7 @@ static noinline btree_path_idx_t btree_paths_realloc(struct btree_trans *trans,
 			  sizeof(struct btree_trans_paths) +
 			  nr * sizeof(struct btree_path) +
 			  nr * sizeof(btree_path_idx_t) + 8 +
-			  nr * sizeof(struct btree_insert_entry), GFP_KERNEL|__GFP_NOFAIL);
+			  nr * sizeof(struct btree_insert_entry), GFP_NOFS|__GFP_NOFAIL);
 
 	unsigned long *paths_allocated = p;
 	memcpy(paths_allocated, trans->paths_allocated, BITS_TO_LONGS(trans->nr_paths) * sizeof(unsigned long));
@@ -3926,7 +3926,7 @@ struct btree_trans *__bch2_trans_get(struct bch_fs *c, unsigned fn_idx)
 		if (s->max_mem) {
 			unsigned expected_mem_bytes = roundup_pow_of_two(s->max_mem);
 
-			trans->mem = kmalloc(expected_mem_bytes, GFP_KERNEL|__GFP_NOWARN);
+			trans->mem = kmalloc(expected_mem_bytes, GFP_NOFS|__GFP_NOWARN);
 			if (likely(trans->mem))
 				trans->mem_bytes = expected_mem_bytes;
 		}
@@ -4229,8 +4229,8 @@ int bch2_fs_btree_iter_init(struct bch_fs *c)
 	if (!c->btree.trans.bufs)
 		return -ENOMEM;
 
-	try(mempool_init_kmalloc_pool(&c->btree.trans.pool, 1, sizeof(struct btree_trans)));
-	try(mempool_init_kmalloc_pool(&c->btree.trans.malloc_pool, 1, BTREE_TRANS_MEM_MAX));
+	try(mempool_init_kmalloc_pool(&c->btree.trans.pool, 8, sizeof(struct btree_trans)));
+	try(mempool_init_kmalloc_pool(&c->btree.trans.malloc_pool, 8, BTREE_TRANS_MEM_MAX));
 	try(init_srcu_struct(&c->btree.trans.barrier));
 
 	/*

fs/data/write.c

@@ -1074,7 +1074,12 @@ static int bch2_write_index_default(struct bch_write_op *op)
 	CLASS(btree_trans, trans)(c);
 
 	struct bkey_buf sk __cleanup(bch2_bkey_buf_exit);
-	bch2_bkey_buf_init(&sk);
+	if (op->prealloc_bkey_buf) {
+		bch2_bkey_buf_init_prealloc(&sk, op->prealloc_bkey_buf);
+		op->prealloc_bkey_buf = NULL;
+	} else {
+		bch2_bkey_buf_init(&sk);
+	}
 
 	do {
 		bch2_trans_begin(trans);
@@ -1457,12 +1462,42 @@ void bch2_write_point_do_index_updates(struct work_struct *work)
 
 		op->flags |= BCH_WRITE_in_worker;
 
+		bool is_swap = op->flags & BCH_WRITE_swap;
+
+		/*
+		 * Pre-allocate the bkey spill buffer while we can still
+		 * do normal allocations.  Use GFP_NOWAIT to avoid entering
+		 * direct reclaim; the kworker context can amplify the
+		 * journal_write -> reclaim -> btree_shrinker -> journal
+		 * deadlock.  If this fails, the old __GFP_NOFAIL path
+		 * under PF_MEMALLOC will handle it.
+		 */
+		if (is_swap && !op->prealloc_bkey_buf)
+			op->prealloc_bkey_buf = kmalloc(2048, GFP_NOWAIT);
+
+		/*
+		 * Swap write ops must not enter direct reclaim:
+		 * we're already in the swap writeback path and reclaim would
+		 * try to swap more pages.
+		 */
+		unsigned int noreclaim_flags = 0;
+		if (is_swap)
+			noreclaim_flags = memalloc_noreclaim_save();
+
 		__bch2_write_index(op);
 
+		if (unlikely(op->prealloc_bkey_buf)) {
+			kfree(op->prealloc_bkey_buf);
+			op->prealloc_bkey_buf = NULL;
+		}
+
 		if (!(op->flags & BCH_WRITE_submitted))
 			__bch2_write(op);
 		else
 			bch2_write_done(op);
+
+		if (is_swap)
+			memalloc_noreclaim_restore(noreclaim_flags);
 	}
 }
 
@@ -2319,7 +2354,14 @@ static void __bch2_write(struct bch_write_op *op)
 		(!(op->flags & BCH_WRITE_submitted) &&
 		 !(op->flags & BCH_WRITE_in_worker));
 
-	guard(memalloc_flags)(PF_MEMALLOC_NOFS);
+	/*
+	 * PF_MEMALLOC_NOFS: prevent filesystem re-entry from allocations.
+	 * For swap writes (BCH_WRITE_swap): also set PF_MEMALLOC to
+	 * prevent entering direct reclaim entirely: swap writes run
+	 * during reclaim and must not recurse into it.
+	 */
+	guard(memalloc_flags)(PF_MEMALLOC_NOFS |
+		((op->flags & BCH_WRITE_swap) ? PF_MEMALLOC : 0));
 
 	if (unlikely(op->opts.nocow &&
 		     c->opts.nocow_enabled) &&

fs/data/write.h

@@ -58,6 +58,7 @@ static inline void bch2_write_op_init(struct bch_write_op *op, struct bch_fs *c,
 	op->new_i_size		= U64_MAX;
 	op->i_sectors_delta	= 0;
 	op->devs_need_flush	= NULL;
+	op->prealloc_bkey_buf	= NULL;
 }
 
 CLOSURE_CALLBACK(bch2_write);

fs/data/write_types.h

@@ -28,7 +28,8 @@
 	x(move)				\
 	x(in_worker)			\
 	x(submitted)			\
-	x(convert_unwritten)
+	x(convert_unwritten)		\
+	x(swap)
 
 enum __bch_write_flags {
 #define x(f)	__BCH_WRITE_##f,
@@ -127,6 +128,8 @@ struct bch_write_op {
 	 */
 	struct bch_devs_mask	*devs_need_flush;
 
+	void			*prealloc_bkey_buf;
+
 	/* Must be last: */
 	struct bch_write_bio	wbio;
 };

fs/vfs/direct.c

@@ -331,7 +331,8 @@ static __always_inline long bch2_dio_write_done(struct dio_write *dio)
 	struct bch_inode_info *inode = dio->inode;
 	bool sync = dio->sync;
 
-	bch2_pagecache_block_put(inode);
+	if (!IS_SWAPFILE(&inode->v))
+		bch2_pagecache_block_put(inode);
 
 	kfree(dio->iov);
 
@@ -464,7 +465,10 @@ static __always_inline long bch2_dio_write_loop(struct dio_write *dio)
 			dio->op.flags |= BCH_WRITE_sync;
 		if (dio->flush)
 			dio->op.flags |= BCH_WRITE_flush;
-		dio->op.flags |= BCH_WRITE_check_enospc;
+		if (IS_SWAPFILE(&inode->v))
+			dio->op.flags |= BCH_WRITE_swap;
+		else
+			dio->op.flags |= BCH_WRITE_check_enospc;
 
 		ret = bch2_quota_reservation_add(c, inode, &dio->quota_res,
 						 bio_sectors(bio), true);
@@ -556,6 +560,17 @@ ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter)
 	if (!enumerated_ref_tryget(&c->writes, BCH_WRITE_REF_dio_write))
 		return -EROFS;
 
+	/*
+	 * Swap I/O: the VFS sets IS_SWAPFILE on the inode during swapon.
+	 * Skip generic_write_checks (rejects swap files with ETXTBSY),
+	 * inode locking, and mtime updates.  Swap I/O is serialized by
+	 * the swap subsystem; PF_MEMALLOC is set by bch2_swap_rw().
+	 */
+	if (IS_SWAPFILE(&inode->v)) {
+		locked = false;
+		goto swap_skip_checks;
+	}
+
 	inode_lock(&inode->v);
 
 	ret = generic_write_checks(req, iter);
@@ -570,17 +585,21 @@ ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter)
 	if (unlikely(ret))
 		goto err_put_write_ref;
 
+swap_skip_checks:
 	if (unlikely((req->ki_pos|iter->count) & (block_bytes(c) - 1))) {
 		ret = bch_err_throw(c, EINVAL_unaligned_io);
 		goto err_put_write_ref;
 	}
 
 	inode_dio_begin(&inode->v);
-	bch2_pagecache_block_get(inode);
+	if (!IS_SWAPFILE(&inode->v))
+		bch2_pagecache_block_get(inode);
 
-	extending = req->ki_pos + iter->count > inode->v.i_size;
+	extending = !IS_SWAPFILE(&inode->v) &&
+		    req->ki_pos + iter->count > inode->v.i_size;
 	if (!extending) {
-		inode_unlock(&inode->v);
+		if (locked)
+			inode_unlock(&inode->v);
 		locked = false;
 	}
 
@@ -604,7 +623,7 @@ ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter)
 	dio->iter		= *iter;
 	dio->op.c		= c;
 
-	if (unlikely(mapping->nrpages)) {
+	if (!IS_SWAPFILE(&inode->v) && unlikely(mapping->nrpages)) {
 		ret = bch2_write_invalidate_inode_pages_range(mapping,
 						req->ki_pos,
 						req->ki_pos + iter->count - 1);
@@ -618,7 +637,8 @@ ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter)
 		inode_unlock(&inode->v);
 	return ret;
 err_put_bio:
-	bch2_pagecache_block_put(inode);
+	if (!IS_SWAPFILE(&inode->v))
+		bch2_pagecache_block_put(inode);
 	bio_put(bio);
 	inode_dio_end(&inode->v);
 err_put_write_ref:

fs/vfs/fs.c

@@ -32,6 +32,7 @@
 #include "vfs/buffered.h"
 #include "vfs/direct.h"
 #include "vfs/pagecache.h"
+#include "vfs/swap.h"
 
 #include <linux/aio.h>
 #include <linux/backing-dev.h>
@@ -1710,6 +1711,9 @@ static const struct address_space_operations bch_address_space_operations = {
 	.migrate_folio	= filemap_migrate_folio,
 #endif
 	.error_remove_folio = generic_error_remove_folio,
+	.swap_activate	= bch2_swap_activate,
+	.swap_deactivate = bch2_swap_deactivate,
+	.swap_rw	= bch2_swap_rw,
 };
 
 struct bcachefs_fid {

fs/vfs/fs.h

@@ -81,6 +81,14 @@ struct bch_inode_info {
 	struct bch_inode_unpacked ei_inode;
 
 	struct delayed_work	ei_writeback_timer;
+
+	/*
+	 * Disk reservation held while this inode is an active swap file.
+	 * Sized to cover the full swap file so ENOSPC can't occur during
+	 * swap COW writes.  Zero when not a swap file.
+	 */
+	u64			ei_swap_reserved_sectors;
+	unsigned		ei_swap_btree_reserve;
 };
 
 #define bch2_pagecache_add_put(i)	bch2_two_state_unlock(&(i)->ei_pagecache_lock, 0)