From 2968ef5e48868fd82ad275fe90a4f8d34aa68267 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Mon, 2 Mar 2026 11:12:24 +0800 Subject: [PATCH 1/2] bcachefs: swap file support via SWP_FS_OPS Add copy-on-write swap file support to bcachefs using the SWP_FS_OPS path, the same mechanism used by network filesystems. This keeps bcachefs in the swap I/O loop so checksumming, encryption, compression, replication, and multi-device placement remain available for swap data. Key design decisions: PF_MEMALLOC/noreclaim context: swap_rw runs under memalloc_noreclaim for both reads and writes. Writes need it because swap writeback runs during reclaim. Reads need it because swap-in page faults can otherwise trigger reclaim that starts competing swap writes. BCH_WRITE_swap propagates the context to the write index kworker and the low-level write path sets PF_MEMALLOC for swap writes. Btree node pinning: at swapon, leaf nodes for extents, inodes, and alloc btrees covering the swap file are marked noevict. This avoids disk reads during memory reclaim; the hot path only does in-memory btree lookups. Btree cache pre-reserve: 16 MB of btree node buffers are pre-allocated on bc->freeable at swapon using GFP_NORETRY to avoid OOM on small VMs. These cover several concurrent fully-cold btree traversals across extents, inodes, and alloc btrees. Disk reservation: swap_pages * PAGE_SECTORS is reserved at swapon to prevent ENOSPC during COW writes. Each swap write allocates a new physical block before freeing the old one; without a reservation, concurrent non-swap writers could consume all free space. Bkey buffer pre-allocation: a 2048-byte buffer is allocated with GFP_NOWAIT before entering noreclaim context, avoiding __GFP_NOFAIL WARN loops when bch2_bkey_buf_realloc spills from the on-stack buffer. IS_SWAPFILE bypass: swap files skip pagecache locking in the direct I/O path because the kernel manages swap file pages directly. This port drops the old review-only debug toggles, the debug-build BUG path, the extents-leaf fill logging, and broad write-buffer PF_MEMALLOC changes from the original draft. A companion ktest scenario is posted as koverstreet/ktest#63. Originally tested at 128-512M RAM with parallel ablation matrix covering pinning on/off, PF_MEMALLOC on/off, btree cache thrashing, drop_caches, multi-device tiered storage, lz4/zstd compression, sustained pressure, concurrent file I/O workload, and disk-throttled I/O. Signed-off-by: Matthias Schorer Signed-off-by: Darafei Praliaskouski (cherry picked from commit eb6062c357d0728da2993375394a6cf6690ec3c5) --- fs/Makefile | 3 +- fs/btree/bkey_buf.h | 6 + fs/btree/cache.c | 72 ++++++++- fs/btree/cache.h | 2 + fs/btree/iter.c | 8 +- fs/data/write.c | 46 +++++- fs/data/write.h | 1 + fs/data/write_types.h | 5 +- fs/vfs/direct.c | 34 ++++- fs/vfs/fs.c | 4 + fs/vfs/fs.h | 8 + fs/vfs/swap.c | 348 ++++++++++++++++++++++++++++++++++++++++++ fs/vfs/swap.h | 16 ++ 13 files changed, 535 insertions(+), 18 deletions(-) create mode 100644 fs/vfs/swap.c create mode 100644 fs/vfs/swap.h diff --git a/fs/Makefile b/fs/Makefile index c4000037b..584922efa 100644 --- a/fs/Makefile +++ b/fs/Makefile @@ -204,7 +204,8 @@ bcachefs-y := \ vfs/io.o \ vfs/buffered.o \ vfs/direct.o \ - vfs/pagecache.o + vfs/pagecache.o \ + vfs/swap.o ifdef CONFIG_DEBUG_FS bcachefs-y += debug/async_objs.o diff --git a/fs/btree/bkey_buf.h b/fs/btree/bkey_buf.h index 7be03293d..9391260e4 100644 --- a/fs/btree/bkey_buf.h +++ b/fs/btree/bkey_buf.h @@ -56,6 +56,12 @@ static inline void bch2_bkey_buf_init(struct bkey_buf *s) bkey_init(&s->k->k); } +static inline void bch2_bkey_buf_init_prealloc(struct bkey_buf *s, void *buf) +{ + s->k = buf; + bkey_init(&s->k->k); +} + static inline void bch2_bkey_buf_exit(struct bkey_buf *s) { if (s->k != (void *) s->onstack) diff --git a/fs/btree/cache.c b/fs/btree/cache.c index 90b344e5f..46089a00a 100644 --- a/fs/btree/cache.c +++ b/fs/btree/cache.c @@ -137,6 +137,67 @@ void bch2_recalc_btree_reserve(struct bch_fs *c) c->btree.cache.nr_reserve = reserve; } +/* + * bch2_btree_cache_add_reserve - pre-allocate btree node buffers for swap I/O + * + * At swapon time, allocate @count empty btree node buffers and put them on + * bc->freeable. bch2_btree_node_mem_alloc() always checks freeable first and + * steals the data buffer from a freeable node rather than going to the page + * allocator — so under PF_MEMALLOC these buffers are used without hitting the + * emergency reserve at all. + * + * A single swap COW write touches at least three btrees (extents, inodes, and + * alloc/freespace), each up to BTREE_MAX_DEPTH levels deep. With mempool=8 + * the worst case is ~24 MB of concurrent node buffers; callers should reserve + * enough to cover their expected concurrency. + * + * We also increment nr_reserve by the same count. nr_reserve is the shrinker + * eviction watermark for bc->live[0]; raising it reduces the shrinker's + * can_free budget, which indirectly shields the freeable pool from being + * drained under memory pressure before swap I/O can use it. + * + * Returns the number of nodes actually allocated (may be less than @count on + * ENOMEM). + */ +static struct btree *__bch2_btree_node_mem_alloc_gfp(struct bch_fs *, gfp_t); + +int bch2_btree_cache_add_reserve(struct bch_fs *c, unsigned count) +{ + struct bch_fs_btree_cache *bc = &c->btree.cache; + unsigned allocated = 0; + + for (unsigned i = 0; i < count; i++) { + struct btree *b = __bch2_btree_node_mem_alloc_gfp(c, + GFP_KERNEL | __GFP_NORETRY); + + if (!b) + break; + + scoped_guard(mutex, &bc->lock) { + bch2_btree_node_transition_state_locked(bc, b, BTREE_NODE_CACHE_FREEABLE); + bc->nr_reserve++; + } + allocated++; + } + + return allocated; +} + +/* + * bch2_btree_cache_remove_reserve - release swap-time btree node reserve + * + * Called at swapoff. Decrements nr_reserve so the shrinker can reclaim the + * extra freeable buffers under normal memory pressure. The actual kvfree of + * the node data buffers happens lazily via the shrinker scan. + */ +void bch2_btree_cache_remove_reserve(struct bch_fs *c, unsigned count) +{ + struct bch_fs_btree_cache *bc = &c->btree.cache; + + scoped_guard(mutex, &bc->lock) + bc->nr_reserve -= min(bc->nr_reserve, (size_t)count); +} + /* Btree node allocation */ struct btree_node_bufs { @@ -280,13 +341,13 @@ static struct btree *__btree_node_mem_alloc(struct bch_fs *c, bool pcpu_read_loc return b; } -struct btree *__bch2_btree_node_mem_alloc(struct bch_fs *c) +static struct btree *__bch2_btree_node_mem_alloc_gfp(struct bch_fs *c, gfp_t gfp) { struct btree_node_bufs bufs = { .byte_order = ilog2(c->opts.btree_node_size) }; struct btree *b; - if (__btree_node_data_alloc(c, &bufs, GFP_KERNEL, false) || - !(b = __btree_node_mem_alloc(c, false, GFP_KERNEL))) { + if (__btree_node_data_alloc(c, &bufs, gfp, false) || + !(b = __btree_node_mem_alloc(c, false, gfp))) { btree_node_bufs_free(&bufs); return NULL; } @@ -296,6 +357,11 @@ struct btree *__bch2_btree_node_mem_alloc(struct bch_fs *c) return b; } +struct btree *__bch2_btree_node_mem_alloc(struct bch_fs *c) +{ + return __bch2_btree_node_mem_alloc_gfp(c, GFP_KERNEL); +} + /* Pinning */ static inline bool __btree_node_pinned(struct bch_fs_btree_cache *bc, struct btree *b) diff --git a/fs/btree/cache.h b/fs/btree/cache.h index 0294e388e..5d16e1d8f 100644 --- a/fs/btree/cache.h +++ b/fs/btree/cache.h @@ -11,6 +11,8 @@ extern const char * const bch2_btree_node_flags[]; struct btree_iter; void bch2_recalc_btree_reserve(struct bch_fs *); +int bch2_btree_cache_add_reserve(struct bch_fs *, unsigned count); +void bch2_btree_cache_remove_reserve(struct bch_fs *, unsigned count); void bch2_btree_node_mem_free(struct bch_fs *, struct btree *); diff --git a/fs/btree/iter.c b/fs/btree/iter.c index 9f10645e9..3133ee6e7 100644 --- a/fs/btree/iter.c +++ b/fs/btree/iter.c @@ -2001,7 +2001,7 @@ static noinline btree_path_idx_t btree_paths_realloc(struct btree_trans *trans, sizeof(struct btree_trans_paths) + nr * sizeof(struct btree_path) + nr * sizeof(btree_path_idx_t) + 8 + - nr * sizeof(struct btree_insert_entry), GFP_KERNEL|__GFP_NOFAIL); + nr * sizeof(struct btree_insert_entry), GFP_NOFS|__GFP_NOFAIL); unsigned long *paths_allocated = p; memcpy(paths_allocated, trans->paths_allocated, BITS_TO_LONGS(trans->nr_paths) * sizeof(unsigned long)); @@ -3926,7 +3926,7 @@ struct btree_trans *__bch2_trans_get(struct bch_fs *c, unsigned fn_idx) if (s->max_mem) { unsigned expected_mem_bytes = roundup_pow_of_two(s->max_mem); - trans->mem = kmalloc(expected_mem_bytes, GFP_KERNEL|__GFP_NOWARN); + trans->mem = kmalloc(expected_mem_bytes, GFP_NOFS|__GFP_NOWARN); if (likely(trans->mem)) trans->mem_bytes = expected_mem_bytes; } @@ -4229,8 +4229,8 @@ int bch2_fs_btree_iter_init(struct bch_fs *c) if (!c->btree.trans.bufs) return -ENOMEM; - try(mempool_init_kmalloc_pool(&c->btree.trans.pool, 1, sizeof(struct btree_trans))); - try(mempool_init_kmalloc_pool(&c->btree.trans.malloc_pool, 1, BTREE_TRANS_MEM_MAX)); + try(mempool_init_kmalloc_pool(&c->btree.trans.pool, 8, sizeof(struct btree_trans))); + try(mempool_init_kmalloc_pool(&c->btree.trans.malloc_pool, 8, BTREE_TRANS_MEM_MAX)); try(init_srcu_struct(&c->btree.trans.barrier)); /* diff --git a/fs/data/write.c b/fs/data/write.c index 59ed55e02..a0c8a5cc9 100644 --- a/fs/data/write.c +++ b/fs/data/write.c @@ -1085,7 +1085,12 @@ static int bch2_write_index_default(struct bch_write_op *op) CLASS(btree_trans, trans)(c); struct bkey_buf sk __cleanup(bch2_bkey_buf_exit); - bch2_bkey_buf_init(&sk); + if (op->prealloc_bkey_buf) { + bch2_bkey_buf_init_prealloc(&sk, op->prealloc_bkey_buf); + op->prealloc_bkey_buf = NULL; + } else { + bch2_bkey_buf_init(&sk); + } do { bch2_trans_begin(trans); @@ -1468,12 +1473,42 @@ void bch2_write_point_do_index_updates(struct work_struct *work) op->flags |= BCH_WRITE_in_worker; + bool is_swap = op->flags & BCH_WRITE_swap; + + /* + * Pre-allocate the bkey spill buffer while we can still + * do normal allocations. Use GFP_NOWAIT to avoid entering + * direct reclaim; the kworker context can amplify the + * journal_write -> reclaim -> btree_shrinker -> journal + * deadlock. If this fails, the old __GFP_NOFAIL path + * under PF_MEMALLOC will handle it. + */ + if (is_swap && !op->prealloc_bkey_buf) + op->prealloc_bkey_buf = kmalloc(2048, GFP_NOWAIT); + + /* + * Swap write ops must not enter direct reclaim: + * we're already in the swap writeback path and reclaim would + * try to swap more pages. + */ + unsigned int noreclaim_flags = 0; + if (is_swap) + noreclaim_flags = memalloc_noreclaim_save(); + __bch2_write_index(op); + if (unlikely(op->prealloc_bkey_buf)) { + kfree(op->prealloc_bkey_buf); + op->prealloc_bkey_buf = NULL; + } + if (!(op->flags & BCH_WRITE_submitted)) __bch2_write(op); else bch2_write_done(op); + + if (is_swap) + memalloc_noreclaim_restore(noreclaim_flags); } } @@ -2330,7 +2365,14 @@ static void __bch2_write(struct bch_write_op *op) (!(op->flags & BCH_WRITE_submitted) && !(op->flags & BCH_WRITE_in_worker)); - guard(memalloc_flags)(PF_MEMALLOC_NOFS); + /* + * PF_MEMALLOC_NOFS: prevent filesystem re-entry from allocations. + * For swap writes (BCH_WRITE_swap): also set PF_MEMALLOC to + * prevent entering direct reclaim entirely: swap writes run + * during reclaim and must not recurse into it. + */ + guard(memalloc_flags)(PF_MEMALLOC_NOFS | + ((op->flags & BCH_WRITE_swap) ? PF_MEMALLOC : 0)); if (unlikely(op->opts.nocow && c->opts.nocow_enabled) && diff --git a/fs/data/write.h b/fs/data/write.h index aaace3a64..2080d4005 100644 --- a/fs/data/write.h +++ b/fs/data/write.h @@ -58,6 +58,7 @@ static inline void bch2_write_op_init(struct bch_write_op *op, struct bch_fs *c, op->new_i_size = U64_MAX; op->i_sectors_delta = 0; op->devs_need_flush = NULL; + op->prealloc_bkey_buf = NULL; } CLOSURE_CALLBACK(bch2_write); diff --git a/fs/data/write_types.h b/fs/data/write_types.h index a15149f7a..9ee083575 100644 --- a/fs/data/write_types.h +++ b/fs/data/write_types.h @@ -28,7 +28,8 @@ x(move) \ x(in_worker) \ x(submitted) \ - x(convert_unwritten) + x(convert_unwritten) \ + x(swap) enum __bch_write_flags { #define x(f) __BCH_WRITE_##f, @@ -127,6 +128,8 @@ struct bch_write_op { */ struct bch_devs_mask *devs_need_flush; + void *prealloc_bkey_buf; + /* Must be last: */ struct bch_write_bio wbio; }; diff --git a/fs/vfs/direct.c b/fs/vfs/direct.c index 6c604f10c..5fd7741fd 100644 --- a/fs/vfs/direct.c +++ b/fs/vfs/direct.c @@ -331,7 +331,8 @@ static __always_inline long bch2_dio_write_done(struct dio_write *dio) struct bch_inode_info *inode = dio->inode; bool sync = dio->sync; - bch2_pagecache_block_put(inode); + if (!IS_SWAPFILE(&inode->v)) + bch2_pagecache_block_put(inode); kfree(dio->iov); @@ -464,7 +465,10 @@ static __always_inline long bch2_dio_write_loop(struct dio_write *dio) dio->op.flags |= BCH_WRITE_sync; if (dio->flush) dio->op.flags |= BCH_WRITE_flush; - dio->op.flags |= BCH_WRITE_check_enospc; + if (IS_SWAPFILE(&inode->v)) + dio->op.flags |= BCH_WRITE_swap; + else + dio->op.flags |= BCH_WRITE_check_enospc; ret = bch2_quota_reservation_add(c, inode, &dio->quota_res, bio_sectors(bio), true); @@ -556,6 +560,17 @@ ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter) if (!enumerated_ref_tryget(&c->writes, BCH_WRITE_REF_dio_write)) return -EROFS; + /* + * Swap I/O: the VFS sets IS_SWAPFILE on the inode during swapon. + * Skip generic_write_checks (rejects swap files with ETXTBSY), + * inode locking, and mtime updates. Swap I/O is serialized by + * the swap subsystem; PF_MEMALLOC is set by bch2_swap_rw(). + */ + if (IS_SWAPFILE(&inode->v)) { + locked = false; + goto swap_skip_checks; + } + inode_lock(&inode->v); ret = generic_write_checks(req, iter); @@ -570,17 +585,21 @@ ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter) if (unlikely(ret)) goto err_put_write_ref; +swap_skip_checks: if (unlikely((req->ki_pos|iter->count) & (block_bytes(c) - 1))) { ret = bch_err_throw(c, EINVAL_unaligned_io); goto err_put_write_ref; } inode_dio_begin(&inode->v); - bch2_pagecache_block_get(inode); + if (!IS_SWAPFILE(&inode->v)) + bch2_pagecache_block_get(inode); - extending = req->ki_pos + iter->count > inode->v.i_size; + extending = !IS_SWAPFILE(&inode->v) && + req->ki_pos + iter->count > inode->v.i_size; if (!extending) { - inode_unlock(&inode->v); + if (locked) + inode_unlock(&inode->v); locked = false; } @@ -604,7 +623,7 @@ ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter) dio->iter = *iter; dio->op.c = c; - if (unlikely(mapping->nrpages)) { + if (!IS_SWAPFILE(&inode->v) && unlikely(mapping->nrpages)) { ret = bch2_write_invalidate_inode_pages_range(mapping, req->ki_pos, req->ki_pos + iter->count - 1); @@ -618,7 +637,8 @@ ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter) inode_unlock(&inode->v); return ret; err_put_bio: - bch2_pagecache_block_put(inode); + if (!IS_SWAPFILE(&inode->v)) + bch2_pagecache_block_put(inode); bio_put(bio); inode_dio_end(&inode->v); err_put_write_ref: diff --git a/fs/vfs/fs.c b/fs/vfs/fs.c index 699f46690..01191f757 100644 --- a/fs/vfs/fs.c +++ b/fs/vfs/fs.c @@ -32,6 +32,7 @@ #include "vfs/buffered.h" #include "vfs/direct.h" #include "vfs/pagecache.h" +#include "vfs/swap.h" #include #include @@ -1918,6 +1919,9 @@ static const struct address_space_operations bch_address_space_operations = { .migrate_folio = filemap_migrate_folio, #endif .error_remove_folio = generic_error_remove_folio, + .swap_activate = bch2_swap_activate, + .swap_deactivate = bch2_swap_deactivate, + .swap_rw = bch2_swap_rw, }; struct bcachefs_fid { diff --git a/fs/vfs/fs.h b/fs/vfs/fs.h index 48b0f8cc5..5053be57e 100644 --- a/fs/vfs/fs.h +++ b/fs/vfs/fs.h @@ -81,6 +81,14 @@ struct bch_inode_info { struct bch_inode_unpacked ei_inode; struct delayed_work ei_writeback_timer; + + /* + * Disk reservation held while this inode is an active swap file. + * Sized to cover the full swap file so ENOSPC can't occur during + * swap COW writes. Zero when not a swap file. + */ + u64 ei_swap_reserved_sectors; + unsigned ei_swap_btree_reserve; }; #define bch2_pagecache_add_put(i) bch2_two_state_unlock(&(i)->ei_pagecache_lock, 0) diff --git a/fs/vfs/swap.c b/fs/vfs/swap.c new file mode 100644 index 000000000..3a3078b3e --- /dev/null +++ b/fs/vfs/swap.c @@ -0,0 +1,348 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef NO_BCACHEFS_FS + +#include "bcachefs.h" +#include "alloc/buckets.h" +#include "btree/cache.h" +#include "btree/iter.h" +#include "btree/update.h" +#include "data/extents.h" +#include "data/io_misc.h" +#include "data/write.h" +#include "sb/members.h" +#include "snapshots/subvolume.h" +#include "vfs/fs.h" +#include "vfs/swap.h" +#include "vfs/direct.h" +#include "vfs/buffered.h" + +#include +#include +#include + +/* + * Swap file support for bcachefs. + * + * Uses the SWP_FS_OPS path (like NFS) so that bcachefs stays in the I/O + * loop for swap operations. This enables checksumming, encryption, + * replication, and multi-device support for swap data. + * + * Key design points: + * - Btree nodes are pinned (noevict) at swapon to avoid disk reads + * during memory reclaim + * - swap I/O runs under noreclaim/PF_MEMALLOC to avoid reclaim re-entry + * - BCH_WRITE_swap flag propagates the same context to the write index + * worker thread + */ + +/* + * Swap I/O diagnostics. + * + * Track in-flight swap ops and warn when one stalls. + */ +static atomic_t bch2_swap_inflight = ATOMIC_INIT(0); +static atomic64_t bch2_swap_completed = ATOMIC64_INIT(0); +static atomic64_t bch2_swap_errors = ATOMIC64_INIT(0); + +/* Warn after 2 s. */ +#define SWAP_IO_WARN_NS (2ULL * NSEC_PER_SEC) + +/* Pin leaf nodes in a btree covering a key range. */ +static int bch2_swap_pin_btree_range(struct btree_trans *trans, + enum btree_id btree, + struct bpos start, struct bpos end, + bool pin) +{ + int count = 0; + + try(for_each_btree_node(trans, iter, btree, + start, 0, BTREE_ITER_prefetch, b, ({ + if (bpos_gt(b->data->min_key, end)) + break; + + if (pin) + set_btree_node_noevict(b); + else + clear_btree_node_noevict(b); + count++; + 0; + }))); + + return count; +} + +static int bch2_swap_pin_extent_alloc_nodes(struct btree_trans *trans, + struct bkey_s_c k, bool pin) +{ + struct bch_fs *c = trans->c; + int total = 0; + + if (!bkey_extent_is_direct_data(k.k)) { + bch_err(c, "swap activate: unsupported extent type %s", + k.k->type < KEY_TYPE_MAX + ? bch2_bkey_types[k.k->type] + : "(unknown)"); + return -EINVAL; + } + + struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); + const union bch_extent_entry *entry; + struct extent_ptr_decoded p; + + bkey_for_each_ptr_decode(k.k, ptrs, p, entry) { + if (p.ptr.dev == BCH_SB_MEMBER_INVALID) + continue; + + CLASS(bch2_dev_bkey_tryget, ca)(c, k, p.ptr.dev); + if (!ca) + return bch_err_throw(c, ENOENT_dev_idx_not_found); + + s64 sectors = ptr_disk_sectors(k.k->size, p); + if (sectors <= 0) + continue; + + struct bpos start = POS(p.ptr.dev, sector_to_bucket(ca, p.ptr.offset)); + struct bpos end = POS(p.ptr.dev, + sector_to_bucket(ca, p.ptr.offset + sectors - 1)); + + if (!bucket_valid(ca, start.offset) || + !bucket_valid(ca, end.offset)) { + bch2_dev_bucket_missing(ca, start.offset); + return -EINVAL; + } + + int ret = bch2_swap_pin_btree_range(trans, BTREE_ID_alloc, + start, end, pin); + if (ret < 0) + return ret; + total += ret; + } + + return total; +} + +static int bch2_swap_pin_alloc_nodes(struct btree_trans *trans, + struct bch_inode_info *inode, + bool pin) +{ + u32 snapshot; + int total = 0; + u64 inum = inode->ei_inum.inum; + + try(bch2_subvolume_get_snapshot(trans, inode->ei_inum.subvol, &snapshot)); + + int ret = for_each_btree_key_max(trans, iter, BTREE_ID_extents, + SPOS(inum, 0, snapshot), SPOS(inum, U64_MAX, snapshot), + BTREE_ITER_prefetch, k, ({ + int nr = bch2_swap_pin_extent_alloc_nodes(trans, k, pin); + if (nr >= 0) + total += nr; + nr < 0 ? nr : 0; + })); + + return ret ?: total; +} + +static int bch2_swap_pin_unpin_nodes(struct bch_fs *c, + struct bch_inode_info *inode, + bool pin) +{ + int total = 0, ret; + + CLASS(btree_trans, trans)(c); + + u64 inum = inode->ei_inum.inum; + + ret = bch2_swap_pin_btree_range(trans, BTREE_ID_extents, + POS(inum, 0), POS(inum, U64_MAX), pin); + if (ret < 0) + return ret; + total += ret; + + ret = bch2_swap_pin_btree_range(trans, BTREE_ID_inodes, + POS(0, inum), POS(0, inum), pin); + if (ret < 0) + return ret; + total += ret; + + ret = bch2_swap_pin_alloc_nodes(trans, inode, pin); + if (ret < 0) + return ret; + total += ret; + + return total; +} + +int bch2_swap_activate(struct swap_info_struct *sis, + struct file *file, sector_t *span) +{ + struct bch_inode_info *inode = file_bch_inode(file); + struct bch_fs *c = inode->v.i_sb->s_fs_info; + + if (!S_ISREG(inode->v.i_mode)) + return -EINVAL; + + /* Pin after prefragmentation (more nodes to pin now). */ + int pinned = bch2_swap_pin_unpin_nodes(c, inode, true); + if (pinned < 0) { + bch_err(c, "swap activate: failed to pin btree nodes: %s", + bch2_err_str(pinned)); + return pinned; + } + + /* + * Reserve disk space for the entire swap file. + * + * Each COW swap write allocates a new physical block before freeing + * the old one. Without a reservation, ENOSPC during reclaim is + * possible if the filesystem is near full, causing swap writes to + * fail, which prevents freeing memory, causing an OOM spiral. + * + * Reserving swap_pages * PAGE_SECTORS at swapon time guarantees + * that space can't be taken by other writers while swap is active. + */ + u64 swap_sectors = (u64)sis->pages * PAGE_SECTORS; + struct disk_reservation disk_res = + bch2_disk_reservation_init(c, c->opts.data_replicas); + int disk_ret = bch2_disk_reservation_get(c, &disk_res, + swap_sectors, + c->opts.data_replicas, 0); + if (disk_ret) { + bch_err(c, "swap activate: insufficient disk space for reservation (%llu sectors, %d replicas): %s", + swap_sectors, c->opts.data_replicas, + bch2_err_str(disk_ret)); + bch2_swap_pin_unpin_nodes(c, inode, false); + return disk_ret; + } + inode->ei_swap_reserved_sectors = disk_res.sectors; + + /* + * Pre-allocate btree node buffers on bc->freeable so that btree reads + * during swap I/O can steal a pre-allocated buffer rather than hitting + * the page allocator under PF_MEMALLOC. + * + * A single swap write traverses extents + inodes + alloc/freespace + * btrees (3 trees * BTREE_MAX_DEPTH levels * 256 KB = 3 MB minimum). + * We reserve 16 MB (= 16 MB / btree_node_size nodes) which covers + * ~5 concurrent fully-cold traversals with headroom. + */ + unsigned swap_reserve = (16 << 20) / c->opts.btree_node_size; + int reserved = bch2_btree_cache_add_reserve(c, swap_reserve); + inode->ei_swap_btree_reserve = reserved; + + sis->flags |= SWP_FS_OPS; + *span = sis->pages; + + bch_info(c, "swap activated on inode %llu (%llu pages, %d nodes pinned, %d/%u btree nodes pre-reserved)", + (u64) inode->v.i_ino, (u64) sis->pages, pinned, reserved, swap_reserve); + + int ret = add_swap_extent(sis, 0, sis->max, 0); + if (ret < 0) { + bch2_btree_cache_remove_reserve(c, inode->ei_swap_btree_reserve); + inode->ei_swap_btree_reserve = 0; + if (inode->ei_swap_reserved_sectors) { + struct disk_reservation dr = { + .sectors = inode->ei_swap_reserved_sectors, + }; + bch2_disk_reservation_put(c, &dr); + inode->ei_swap_reserved_sectors = 0; + } + bch2_swap_pin_unpin_nodes(c, inode, false); + } + return ret; +} + +void bch2_swap_deactivate(struct file *file) +{ + struct bch_inode_info *inode = file_bch_inode(file); + struct bch_fs *c = inode->v.i_sb->s_fs_info; + + bch2_swap_pin_unpin_nodes(c, inode, false); + + bch2_btree_cache_remove_reserve(c, inode->ei_swap_btree_reserve); + inode->ei_swap_btree_reserve = 0; + + if (inode->ei_swap_reserved_sectors) { + struct disk_reservation disk_res = { + .sectors = inode->ei_swap_reserved_sectors, + }; + bch2_disk_reservation_put(c, &disk_res); + inode->ei_swap_reserved_sectors = 0; + } + + bch_info(c, "swap deactivated on inode %llu", (u64) inode->v.i_ino); +} + +/* + * Swap I/O callback: called for every swap read/write when SWP_FS_OPS + * is set. Returns bytes transferred or -EIOCBQUEUED for async I/O. + */ +int bch2_swap_rw(struct kiocb *iocb, struct iov_iter *iter) +{ + struct bch_fs *c = file_inode(iocb->ki_filp)->i_sb->s_fs_info; + u64 start_ns = ktime_get_ns(); + int rw = iov_iter_rw(iter); + + atomic_inc(&bch2_swap_inflight); + + iocb->ki_flags |= IOCB_DIRECT; + + /* + * Prevent reclaim re-entry for both writes and reads. + * + * Writes: swap writeback runs during reclaim, so allocations in + * the write path must not trigger reclaim (circular dependency). + * + * Reads: swap-in happens during page fault. If a read-path + * allocation enters reclaim, reclaim can try to swap out other + * pages; those writes compete for the same btree locks as the read. + * + * PF_MEMALLOC bypasses watermarks and skips direct reclaim. + */ + unsigned int noreclaim_flags = memalloc_noreclaim_save(); + + ssize_t ret; + if (rw == READ) + ret = bch2_read_iter(iocb, iter); + else + ret = bch2_write_iter(iocb, iter); + + memalloc_noreclaim_restore(noreclaim_flags); + + atomic_dec(&bch2_swap_inflight); + + u64 elapsed_ns = ktime_get_ns() - start_ns; + + if (ret < 0 && ret != -EIOCBQUEUED) { + atomic64_inc(&bch2_swap_errors); + bch_err_ratelimited(c, "swap_rw %s error %li at pos %lld " + "(inflight=%d completed=%lld errors=%lld)", + rw == READ ? "read" : "write", + ret, iocb->ki_pos, + atomic_read(&bch2_swap_inflight), + atomic64_read(&bch2_swap_completed), + atomic64_read(&bch2_swap_errors)); + } else { + atomic64_inc(&bch2_swap_completed); + } + + /* + * Detect stalled swap I/O. If a single operation takes more than 2 s, + * something is badly wrong, likely PF_MEMALLOC reserves exhausted or a + * deadlock. + */ + if (unlikely(elapsed_ns > SWAP_IO_WARN_NS)) { + bch_err(c, "swap_rw %s STALL: %llu ms at pos %lld " + "(inflight=%d completed=%lld errors=%lld)", + rw == READ ? "read" : "write", + elapsed_ns / NSEC_PER_MSEC, iocb->ki_pos, + atomic_read(&bch2_swap_inflight), + atomic64_read(&bch2_swap_completed), + atomic64_read(&bch2_swap_errors)); + WARN_ON_ONCE(1); + } + + return ret; +} + +#endif /* NO_BCACHEFS_FS */ diff --git a/fs/vfs/swap.h b/fs/vfs/swap.h new file mode 100644 index 000000000..50e06967c --- /dev/null +++ b/fs/vfs/swap.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _BCACHEFS_VFS_SWAP_H +#define _BCACHEFS_VFS_SWAP_H + +#ifndef NO_BCACHEFS_FS + +struct swap_info_struct; + +int bch2_swap_activate(struct swap_info_struct *, struct file *, sector_t *); +void bch2_swap_deactivate(struct file *); +int bch2_swap_rw(struct kiocb *, struct iov_iter *); + +#else + +#endif /* NO_BCACHEFS_FS */ +#endif /* _BCACHEFS_VFS_SWAP_H */ From 6472884b537e0fc44befd6c9e6d8002969fd087f Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Tue, 7 Jul 2026 14:55:57 +0400 Subject: [PATCH 2/2] bcachefs: simplify swapfile fs-ops support --- fs/btree/bkey_buf.h | 6 - fs/btree/cache.c | 72 +--------- fs/btree/cache.h | 2 - fs/btree/iter.c | 8 +- fs/data/write.c | 46 +----- fs/data/write.h | 1 - fs/data/write_types.h | 5 +- fs/vfs/direct.c | 53 +++---- fs/vfs/direct.h | 3 +- fs/vfs/fs.h | 8 -- fs/vfs/swap.c | 321 +----------------------------------------- fs/vfs/swap.h | 4 +- 12 files changed, 42 insertions(+), 487 deletions(-) diff --git a/fs/btree/bkey_buf.h b/fs/btree/bkey_buf.h index 9391260e4..7be03293d 100644 --- a/fs/btree/bkey_buf.h +++ b/fs/btree/bkey_buf.h @@ -56,12 +56,6 @@ static inline void bch2_bkey_buf_init(struct bkey_buf *s) bkey_init(&s->k->k); } -static inline void bch2_bkey_buf_init_prealloc(struct bkey_buf *s, void *buf) -{ - s->k = buf; - bkey_init(&s->k->k); -} - static inline void bch2_bkey_buf_exit(struct bkey_buf *s) { if (s->k != (void *) s->onstack) diff --git a/fs/btree/cache.c b/fs/btree/cache.c index 46089a00a..90b344e5f 100644 --- a/fs/btree/cache.c +++ b/fs/btree/cache.c @@ -137,67 +137,6 @@ void bch2_recalc_btree_reserve(struct bch_fs *c) c->btree.cache.nr_reserve = reserve; } -/* - * bch2_btree_cache_add_reserve - pre-allocate btree node buffers for swap I/O - * - * At swapon time, allocate @count empty btree node buffers and put them on - * bc->freeable. bch2_btree_node_mem_alloc() always checks freeable first and - * steals the data buffer from a freeable node rather than going to the page - * allocator — so under PF_MEMALLOC these buffers are used without hitting the - * emergency reserve at all. - * - * A single swap COW write touches at least three btrees (extents, inodes, and - * alloc/freespace), each up to BTREE_MAX_DEPTH levels deep. With mempool=8 - * the worst case is ~24 MB of concurrent node buffers; callers should reserve - * enough to cover their expected concurrency. - * - * We also increment nr_reserve by the same count. nr_reserve is the shrinker - * eviction watermark for bc->live[0]; raising it reduces the shrinker's - * can_free budget, which indirectly shields the freeable pool from being - * drained under memory pressure before swap I/O can use it. - * - * Returns the number of nodes actually allocated (may be less than @count on - * ENOMEM). - */ -static struct btree *__bch2_btree_node_mem_alloc_gfp(struct bch_fs *, gfp_t); - -int bch2_btree_cache_add_reserve(struct bch_fs *c, unsigned count) -{ - struct bch_fs_btree_cache *bc = &c->btree.cache; - unsigned allocated = 0; - - for (unsigned i = 0; i < count; i++) { - struct btree *b = __bch2_btree_node_mem_alloc_gfp(c, - GFP_KERNEL | __GFP_NORETRY); - - if (!b) - break; - - scoped_guard(mutex, &bc->lock) { - bch2_btree_node_transition_state_locked(bc, b, BTREE_NODE_CACHE_FREEABLE); - bc->nr_reserve++; - } - allocated++; - } - - return allocated; -} - -/* - * bch2_btree_cache_remove_reserve - release swap-time btree node reserve - * - * Called at swapoff. Decrements nr_reserve so the shrinker can reclaim the - * extra freeable buffers under normal memory pressure. The actual kvfree of - * the node data buffers happens lazily via the shrinker scan. - */ -void bch2_btree_cache_remove_reserve(struct bch_fs *c, unsigned count) -{ - struct bch_fs_btree_cache *bc = &c->btree.cache; - - scoped_guard(mutex, &bc->lock) - bc->nr_reserve -= min(bc->nr_reserve, (size_t)count); -} - /* Btree node allocation */ struct btree_node_bufs { @@ -341,13 +280,13 @@ static struct btree *__btree_node_mem_alloc(struct bch_fs *c, bool pcpu_read_loc return b; } -static struct btree *__bch2_btree_node_mem_alloc_gfp(struct bch_fs *c, gfp_t gfp) +struct btree *__bch2_btree_node_mem_alloc(struct bch_fs *c) { struct btree_node_bufs bufs = { .byte_order = ilog2(c->opts.btree_node_size) }; struct btree *b; - if (__btree_node_data_alloc(c, &bufs, gfp, false) || - !(b = __btree_node_mem_alloc(c, false, gfp))) { + if (__btree_node_data_alloc(c, &bufs, GFP_KERNEL, false) || + !(b = __btree_node_mem_alloc(c, false, GFP_KERNEL))) { btree_node_bufs_free(&bufs); return NULL; } @@ -357,11 +296,6 @@ static struct btree *__bch2_btree_node_mem_alloc_gfp(struct bch_fs *c, gfp_t gfp return b; } -struct btree *__bch2_btree_node_mem_alloc(struct bch_fs *c) -{ - return __bch2_btree_node_mem_alloc_gfp(c, GFP_KERNEL); -} - /* Pinning */ static inline bool __btree_node_pinned(struct bch_fs_btree_cache *bc, struct btree *b) diff --git a/fs/btree/cache.h b/fs/btree/cache.h index 5d16e1d8f..0294e388e 100644 --- a/fs/btree/cache.h +++ b/fs/btree/cache.h @@ -11,8 +11,6 @@ extern const char * const bch2_btree_node_flags[]; struct btree_iter; void bch2_recalc_btree_reserve(struct bch_fs *); -int bch2_btree_cache_add_reserve(struct bch_fs *, unsigned count); -void bch2_btree_cache_remove_reserve(struct bch_fs *, unsigned count); void bch2_btree_node_mem_free(struct bch_fs *, struct btree *); diff --git a/fs/btree/iter.c b/fs/btree/iter.c index 3133ee6e7..9f10645e9 100644 --- a/fs/btree/iter.c +++ b/fs/btree/iter.c @@ -2001,7 +2001,7 @@ static noinline btree_path_idx_t btree_paths_realloc(struct btree_trans *trans, sizeof(struct btree_trans_paths) + nr * sizeof(struct btree_path) + nr * sizeof(btree_path_idx_t) + 8 + - nr * sizeof(struct btree_insert_entry), GFP_NOFS|__GFP_NOFAIL); + nr * sizeof(struct btree_insert_entry), GFP_KERNEL|__GFP_NOFAIL); unsigned long *paths_allocated = p; memcpy(paths_allocated, trans->paths_allocated, BITS_TO_LONGS(trans->nr_paths) * sizeof(unsigned long)); @@ -3926,7 +3926,7 @@ struct btree_trans *__bch2_trans_get(struct bch_fs *c, unsigned fn_idx) if (s->max_mem) { unsigned expected_mem_bytes = roundup_pow_of_two(s->max_mem); - trans->mem = kmalloc(expected_mem_bytes, GFP_NOFS|__GFP_NOWARN); + trans->mem = kmalloc(expected_mem_bytes, GFP_KERNEL|__GFP_NOWARN); if (likely(trans->mem)) trans->mem_bytes = expected_mem_bytes; } @@ -4229,8 +4229,8 @@ int bch2_fs_btree_iter_init(struct bch_fs *c) if (!c->btree.trans.bufs) return -ENOMEM; - try(mempool_init_kmalloc_pool(&c->btree.trans.pool, 8, sizeof(struct btree_trans))); - try(mempool_init_kmalloc_pool(&c->btree.trans.malloc_pool, 8, BTREE_TRANS_MEM_MAX)); + try(mempool_init_kmalloc_pool(&c->btree.trans.pool, 1, sizeof(struct btree_trans))); + try(mempool_init_kmalloc_pool(&c->btree.trans.malloc_pool, 1, BTREE_TRANS_MEM_MAX)); try(init_srcu_struct(&c->btree.trans.barrier)); /* diff --git a/fs/data/write.c b/fs/data/write.c index a0c8a5cc9..59ed55e02 100644 --- a/fs/data/write.c +++ b/fs/data/write.c @@ -1085,12 +1085,7 @@ static int bch2_write_index_default(struct bch_write_op *op) CLASS(btree_trans, trans)(c); struct bkey_buf sk __cleanup(bch2_bkey_buf_exit); - if (op->prealloc_bkey_buf) { - bch2_bkey_buf_init_prealloc(&sk, op->prealloc_bkey_buf); - op->prealloc_bkey_buf = NULL; - } else { - bch2_bkey_buf_init(&sk); - } + bch2_bkey_buf_init(&sk); do { bch2_trans_begin(trans); @@ -1473,42 +1468,12 @@ void bch2_write_point_do_index_updates(struct work_struct *work) op->flags |= BCH_WRITE_in_worker; - bool is_swap = op->flags & BCH_WRITE_swap; - - /* - * Pre-allocate the bkey spill buffer while we can still - * do normal allocations. Use GFP_NOWAIT to avoid entering - * direct reclaim; the kworker context can amplify the - * journal_write -> reclaim -> btree_shrinker -> journal - * deadlock. If this fails, the old __GFP_NOFAIL path - * under PF_MEMALLOC will handle it. - */ - if (is_swap && !op->prealloc_bkey_buf) - op->prealloc_bkey_buf = kmalloc(2048, GFP_NOWAIT); - - /* - * Swap write ops must not enter direct reclaim: - * we're already in the swap writeback path and reclaim would - * try to swap more pages. - */ - unsigned int noreclaim_flags = 0; - if (is_swap) - noreclaim_flags = memalloc_noreclaim_save(); - __bch2_write_index(op); - if (unlikely(op->prealloc_bkey_buf)) { - kfree(op->prealloc_bkey_buf); - op->prealloc_bkey_buf = NULL; - } - if (!(op->flags & BCH_WRITE_submitted)) __bch2_write(op); else bch2_write_done(op); - - if (is_swap) - memalloc_noreclaim_restore(noreclaim_flags); } } @@ -2365,14 +2330,7 @@ static void __bch2_write(struct bch_write_op *op) (!(op->flags & BCH_WRITE_submitted) && !(op->flags & BCH_WRITE_in_worker)); - /* - * PF_MEMALLOC_NOFS: prevent filesystem re-entry from allocations. - * For swap writes (BCH_WRITE_swap): also set PF_MEMALLOC to - * prevent entering direct reclaim entirely: swap writes run - * during reclaim and must not recurse into it. - */ - guard(memalloc_flags)(PF_MEMALLOC_NOFS | - ((op->flags & BCH_WRITE_swap) ? PF_MEMALLOC : 0)); + guard(memalloc_flags)(PF_MEMALLOC_NOFS); if (unlikely(op->opts.nocow && c->opts.nocow_enabled) && diff --git a/fs/data/write.h b/fs/data/write.h index 2080d4005..aaace3a64 100644 --- a/fs/data/write.h +++ b/fs/data/write.h @@ -58,7 +58,6 @@ static inline void bch2_write_op_init(struct bch_write_op *op, struct bch_fs *c, op->new_i_size = U64_MAX; op->i_sectors_delta = 0; op->devs_need_flush = NULL; - op->prealloc_bkey_buf = NULL; } CLOSURE_CALLBACK(bch2_write); diff --git a/fs/data/write_types.h b/fs/data/write_types.h index 9ee083575..a15149f7a 100644 --- a/fs/data/write_types.h +++ b/fs/data/write_types.h @@ -28,8 +28,7 @@ x(move) \ x(in_worker) \ x(submitted) \ - x(convert_unwritten) \ - x(swap) + x(convert_unwritten) enum __bch_write_flags { #define x(f) __BCH_WRITE_##f, @@ -128,8 +127,6 @@ struct bch_write_op { */ struct bch_devs_mask *devs_need_flush; - void *prealloc_bkey_buf; - /* Must be last: */ struct bch_write_bio wbio; }; diff --git a/fs/vfs/direct.c b/fs/vfs/direct.c index 5fd7741fd..711ed8fdf 100644 --- a/fs/vfs/direct.c +++ b/fs/vfs/direct.c @@ -331,7 +331,7 @@ static __always_inline long bch2_dio_write_done(struct dio_write *dio) struct bch_inode_info *inode = dio->inode; bool sync = dio->sync; - if (!IS_SWAPFILE(&inode->v)) + if (dio->pagecache_blocked) bch2_pagecache_block_put(inode); kfree(dio->iov); @@ -465,10 +465,7 @@ static __always_inline long bch2_dio_write_loop(struct dio_write *dio) dio->op.flags |= BCH_WRITE_sync; if (dio->flush) dio->op.flags |= BCH_WRITE_flush; - if (IS_SWAPFILE(&inode->v)) - dio->op.flags |= BCH_WRITE_swap; - else - dio->op.flags |= BCH_WRITE_check_enospc; + dio->op.flags |= BCH_WRITE_check_enospc; ret = bch2_quota_reservation_add(c, inode, &dio->quota_res, bio_sectors(bio), true); @@ -550,6 +547,7 @@ ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter) struct dio_write *dio; struct bio *bio; bool locked = true, extending; + bool swap = IS_SWAPFILE(&inode->v); ssize_t ret; prefetch(&c->opts); @@ -560,43 +558,35 @@ ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter) if (!enumerated_ref_tryget(&c->writes, BCH_WRITE_REF_dio_write)) return -EROFS; - /* - * Swap I/O: the VFS sets IS_SWAPFILE on the inode during swapon. - * Skip generic_write_checks (rejects swap files with ETXTBSY), - * inode locking, and mtime updates. Swap I/O is serialized by - * the swap subsystem; PF_MEMALLOC is set by bch2_swap_rw(). - */ - if (IS_SWAPFILE(&inode->v)) { + if (!swap) + inode_lock(&inode->v); + else locked = false; - goto swap_skip_checks; - } - - inode_lock(&inode->v); - ret = generic_write_checks(req, iter); - if (unlikely(ret <= 0)) - goto err_put_write_ref; + if (!swap) { + ret = generic_write_checks(req, iter); + if (unlikely(ret <= 0)) + goto err_put_write_ref; - ret = file_remove_privs(file); - if (unlikely(ret)) - goto err_put_write_ref; + ret = file_remove_privs(file); + if (unlikely(ret)) + goto err_put_write_ref; - ret = file_update_time(file); - if (unlikely(ret)) - goto err_put_write_ref; + ret = file_update_time(file); + if (unlikely(ret)) + goto err_put_write_ref; + } -swap_skip_checks: if (unlikely((req->ki_pos|iter->count) & (block_bytes(c) - 1))) { ret = bch_err_throw(c, EINVAL_unaligned_io); goto err_put_write_ref; } inode_dio_begin(&inode->v); - if (!IS_SWAPFILE(&inode->v)) + if (!swap) bch2_pagecache_block_get(inode); - extending = !IS_SWAPFILE(&inode->v) && - req->ki_pos + iter->count > inode->v.i_size; + extending = !swap && req->ki_pos + iter->count > inode->v.i_size; if (!extending) { if (locked) inode_unlock(&inode->v); @@ -618,12 +608,13 @@ ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter) dio->extending = extending; dio->sync = is_sync_kiocb(req) || extending; dio->flush = iocb_is_dsync(req) && !c->opts.journal_flush_disabled; + dio->pagecache_blocked = !swap; dio->quota_res.sectors = 0; dio->written = 0; dio->iter = *iter; dio->op.c = c; - if (!IS_SWAPFILE(&inode->v) && unlikely(mapping->nrpages)) { + if (!swap && unlikely(mapping->nrpages)) { ret = bch2_write_invalidate_inode_pages_range(mapping, req->ki_pos, req->ki_pos + iter->count - 1); @@ -637,7 +628,7 @@ ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter) inode_unlock(&inode->v); return ret; err_put_bio: - if (!IS_SWAPFILE(&inode->v)) + if (!swap) bch2_pagecache_block_put(inode); bio_put(bio); inode_dio_end(&inode->v); diff --git a/fs/vfs/direct.h b/fs/vfs/direct.h index 9bffbbbcf..0a28a7969 100644 --- a/fs/vfs/direct.h +++ b/fs/vfs/direct.h @@ -24,7 +24,8 @@ struct dio_write { extending:1, sync:1, sync_done:1, - flush:1; + flush:1, + pagecache_blocked:1; struct quota_res quota_res; u64 written; diff --git a/fs/vfs/fs.h b/fs/vfs/fs.h index 5053be57e..48b0f8cc5 100644 --- a/fs/vfs/fs.h +++ b/fs/vfs/fs.h @@ -81,14 +81,6 @@ struct bch_inode_info { struct bch_inode_unpacked ei_inode; struct delayed_work ei_writeback_timer; - - /* - * Disk reservation held while this inode is an active swap file. - * Sized to cover the full swap file so ENOSPC can't occur during - * swap COW writes. Zero when not a swap file. - */ - u64 ei_swap_reserved_sectors; - unsigned ei_swap_btree_reserve; }; #define bch2_pagecache_add_put(i) bch2_two_state_unlock(&(i)->ei_pagecache_lock, 0) diff --git a/fs/vfs/swap.c b/fs/vfs/swap.c index 3a3078b3e..371d5e07c 100644 --- a/fs/vfs/swap.c +++ b/fs/vfs/swap.c @@ -2,347 +2,38 @@ #ifndef NO_BCACHEFS_FS #include "bcachefs.h" -#include "alloc/buckets.h" -#include "btree/cache.h" -#include "btree/iter.h" -#include "btree/update.h" -#include "data/extents.h" -#include "data/io_misc.h" -#include "data/write.h" -#include "sb/members.h" -#include "snapshots/subvolume.h" +#include "vfs/buffered.h" +#include "vfs/direct.h" #include "vfs/fs.h" #include "vfs/swap.h" -#include "vfs/direct.h" -#include "vfs/buffered.h" -#include #include -#include - -/* - * Swap file support for bcachefs. - * - * Uses the SWP_FS_OPS path (like NFS) so that bcachefs stays in the I/O - * loop for swap operations. This enables checksumming, encryption, - * replication, and multi-device support for swap data. - * - * Key design points: - * - Btree nodes are pinned (noevict) at swapon to avoid disk reads - * during memory reclaim - * - swap I/O runs under noreclaim/PF_MEMALLOC to avoid reclaim re-entry - * - BCH_WRITE_swap flag propagates the same context to the write index - * worker thread - */ - -/* - * Swap I/O diagnostics. - * - * Track in-flight swap ops and warn when one stalls. - */ -static atomic_t bch2_swap_inflight = ATOMIC_INIT(0); -static atomic64_t bch2_swap_completed = ATOMIC64_INIT(0); -static atomic64_t bch2_swap_errors = ATOMIC64_INIT(0); - -/* Warn after 2 s. */ -#define SWAP_IO_WARN_NS (2ULL * NSEC_PER_SEC) - -/* Pin leaf nodes in a btree covering a key range. */ -static int bch2_swap_pin_btree_range(struct btree_trans *trans, - enum btree_id btree, - struct bpos start, struct bpos end, - bool pin) -{ - int count = 0; - - try(for_each_btree_node(trans, iter, btree, - start, 0, BTREE_ITER_prefetch, b, ({ - if (bpos_gt(b->data->min_key, end)) - break; - - if (pin) - set_btree_node_noevict(b); - else - clear_btree_node_noevict(b); - count++; - 0; - }))); - - return count; -} - -static int bch2_swap_pin_extent_alloc_nodes(struct btree_trans *trans, - struct bkey_s_c k, bool pin) -{ - struct bch_fs *c = trans->c; - int total = 0; - - if (!bkey_extent_is_direct_data(k.k)) { - bch_err(c, "swap activate: unsupported extent type %s", - k.k->type < KEY_TYPE_MAX - ? bch2_bkey_types[k.k->type] - : "(unknown)"); - return -EINVAL; - } - - struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); - const union bch_extent_entry *entry; - struct extent_ptr_decoded p; - - bkey_for_each_ptr_decode(k.k, ptrs, p, entry) { - if (p.ptr.dev == BCH_SB_MEMBER_INVALID) - continue; - - CLASS(bch2_dev_bkey_tryget, ca)(c, k, p.ptr.dev); - if (!ca) - return bch_err_throw(c, ENOENT_dev_idx_not_found); - - s64 sectors = ptr_disk_sectors(k.k->size, p); - if (sectors <= 0) - continue; - - struct bpos start = POS(p.ptr.dev, sector_to_bucket(ca, p.ptr.offset)); - struct bpos end = POS(p.ptr.dev, - sector_to_bucket(ca, p.ptr.offset + sectors - 1)); - - if (!bucket_valid(ca, start.offset) || - !bucket_valid(ca, end.offset)) { - bch2_dev_bucket_missing(ca, start.offset); - return -EINVAL; - } - - int ret = bch2_swap_pin_btree_range(trans, BTREE_ID_alloc, - start, end, pin); - if (ret < 0) - return ret; - total += ret; - } - - return total; -} - -static int bch2_swap_pin_alloc_nodes(struct btree_trans *trans, - struct bch_inode_info *inode, - bool pin) -{ - u32 snapshot; - int total = 0; - u64 inum = inode->ei_inum.inum; - - try(bch2_subvolume_get_snapshot(trans, inode->ei_inum.subvol, &snapshot)); - - int ret = for_each_btree_key_max(trans, iter, BTREE_ID_extents, - SPOS(inum, 0, snapshot), SPOS(inum, U64_MAX, snapshot), - BTREE_ITER_prefetch, k, ({ - int nr = bch2_swap_pin_extent_alloc_nodes(trans, k, pin); - if (nr >= 0) - total += nr; - nr < 0 ? nr : 0; - })); - - return ret ?: total; -} - -static int bch2_swap_pin_unpin_nodes(struct bch_fs *c, - struct bch_inode_info *inode, - bool pin) -{ - int total = 0, ret; - - CLASS(btree_trans, trans)(c); - - u64 inum = inode->ei_inum.inum; - - ret = bch2_swap_pin_btree_range(trans, BTREE_ID_extents, - POS(inum, 0), POS(inum, U64_MAX), pin); - if (ret < 0) - return ret; - total += ret; - - ret = bch2_swap_pin_btree_range(trans, BTREE_ID_inodes, - POS(0, inum), POS(0, inum), pin); - if (ret < 0) - return ret; - total += ret; - - ret = bch2_swap_pin_alloc_nodes(trans, inode, pin); - if (ret < 0) - return ret; - total += ret; - - return total; -} int bch2_swap_activate(struct swap_info_struct *sis, struct file *file, sector_t *span) { struct bch_inode_info *inode = file_bch_inode(file); - struct bch_fs *c = inode->v.i_sb->s_fs_info; if (!S_ISREG(inode->v.i_mode)) return -EINVAL; - /* Pin after prefragmentation (more nodes to pin now). */ - int pinned = bch2_swap_pin_unpin_nodes(c, inode, true); - if (pinned < 0) { - bch_err(c, "swap activate: failed to pin btree nodes: %s", - bch2_err_str(pinned)); - return pinned; - } - - /* - * Reserve disk space for the entire swap file. - * - * Each COW swap write allocates a new physical block before freeing - * the old one. Without a reservation, ENOSPC during reclaim is - * possible if the filesystem is near full, causing swap writes to - * fail, which prevents freeing memory, causing an OOM spiral. - * - * Reserving swap_pages * PAGE_SECTORS at swapon time guarantees - * that space can't be taken by other writers while swap is active. - */ - u64 swap_sectors = (u64)sis->pages * PAGE_SECTORS; - struct disk_reservation disk_res = - bch2_disk_reservation_init(c, c->opts.data_replicas); - int disk_ret = bch2_disk_reservation_get(c, &disk_res, - swap_sectors, - c->opts.data_replicas, 0); - if (disk_ret) { - bch_err(c, "swap activate: insufficient disk space for reservation (%llu sectors, %d replicas): %s", - swap_sectors, c->opts.data_replicas, - bch2_err_str(disk_ret)); - bch2_swap_pin_unpin_nodes(c, inode, false); - return disk_ret; - } - inode->ei_swap_reserved_sectors = disk_res.sectors; - - /* - * Pre-allocate btree node buffers on bc->freeable so that btree reads - * during swap I/O can steal a pre-allocated buffer rather than hitting - * the page allocator under PF_MEMALLOC. - * - * A single swap write traverses extents + inodes + alloc/freespace - * btrees (3 trees * BTREE_MAX_DEPTH levels * 256 KB = 3 MB minimum). - * We reserve 16 MB (= 16 MB / btree_node_size nodes) which covers - * ~5 concurrent fully-cold traversals with headroom. - */ - unsigned swap_reserve = (16 << 20) / c->opts.btree_node_size; - int reserved = bch2_btree_cache_add_reserve(c, swap_reserve); - inode->ei_swap_btree_reserve = reserved; - sis->flags |= SWP_FS_OPS; *span = sis->pages; - bch_info(c, "swap activated on inode %llu (%llu pages, %d nodes pinned, %d/%u btree nodes pre-reserved)", - (u64) inode->v.i_ino, (u64) sis->pages, pinned, reserved, swap_reserve); - - int ret = add_swap_extent(sis, 0, sis->max, 0); - if (ret < 0) { - bch2_btree_cache_remove_reserve(c, inode->ei_swap_btree_reserve); - inode->ei_swap_btree_reserve = 0; - if (inode->ei_swap_reserved_sectors) { - struct disk_reservation dr = { - .sectors = inode->ei_swap_reserved_sectors, - }; - bch2_disk_reservation_put(c, &dr); - inode->ei_swap_reserved_sectors = 0; - } - bch2_swap_pin_unpin_nodes(c, inode, false); - } - return ret; + return add_swap_extent(sis, 0, sis->max, 0); } void bch2_swap_deactivate(struct file *file) { - struct bch_inode_info *inode = file_bch_inode(file); - struct bch_fs *c = inode->v.i_sb->s_fs_info; - - bch2_swap_pin_unpin_nodes(c, inode, false); - - bch2_btree_cache_remove_reserve(c, inode->ei_swap_btree_reserve); - inode->ei_swap_btree_reserve = 0; - - if (inode->ei_swap_reserved_sectors) { - struct disk_reservation disk_res = { - .sectors = inode->ei_swap_reserved_sectors, - }; - bch2_disk_reservation_put(c, &disk_res); - inode->ei_swap_reserved_sectors = 0; - } - - bch_info(c, "swap deactivated on inode %llu", (u64) inode->v.i_ino); } -/* - * Swap I/O callback: called for every swap read/write when SWP_FS_OPS - * is set. Returns bytes transferred or -EIOCBQUEUED for async I/O. - */ int bch2_swap_rw(struct kiocb *iocb, struct iov_iter *iter) { - struct bch_fs *c = file_inode(iocb->ki_filp)->i_sb->s_fs_info; - u64 start_ns = ktime_get_ns(); - int rw = iov_iter_rw(iter); - - atomic_inc(&bch2_swap_inflight); - iocb->ki_flags |= IOCB_DIRECT; - /* - * Prevent reclaim re-entry for both writes and reads. - * - * Writes: swap writeback runs during reclaim, so allocations in - * the write path must not trigger reclaim (circular dependency). - * - * Reads: swap-in happens during page fault. If a read-path - * allocation enters reclaim, reclaim can try to swap out other - * pages; those writes compete for the same btree locks as the read. - * - * PF_MEMALLOC bypasses watermarks and skips direct reclaim. - */ - unsigned int noreclaim_flags = memalloc_noreclaim_save(); - - ssize_t ret; - if (rw == READ) - ret = bch2_read_iter(iocb, iter); - else - ret = bch2_write_iter(iocb, iter); - - memalloc_noreclaim_restore(noreclaim_flags); - - atomic_dec(&bch2_swap_inflight); - - u64 elapsed_ns = ktime_get_ns() - start_ns; - - if (ret < 0 && ret != -EIOCBQUEUED) { - atomic64_inc(&bch2_swap_errors); - bch_err_ratelimited(c, "swap_rw %s error %li at pos %lld " - "(inflight=%d completed=%lld errors=%lld)", - rw == READ ? "read" : "write", - ret, iocb->ki_pos, - atomic_read(&bch2_swap_inflight), - atomic64_read(&bch2_swap_completed), - atomic64_read(&bch2_swap_errors)); - } else { - atomic64_inc(&bch2_swap_completed); - } - - /* - * Detect stalled swap I/O. If a single operation takes more than 2 s, - * something is badly wrong, likely PF_MEMALLOC reserves exhausted or a - * deadlock. - */ - if (unlikely(elapsed_ns > SWAP_IO_WARN_NS)) { - bch_err(c, "swap_rw %s STALL: %llu ms at pos %lld " - "(inflight=%d completed=%lld errors=%lld)", - rw == READ ? "read" : "write", - elapsed_ns / NSEC_PER_MSEC, iocb->ki_pos, - atomic_read(&bch2_swap_inflight), - atomic64_read(&bch2_swap_completed), - atomic64_read(&bch2_swap_errors)); - WARN_ON_ONCE(1); - } - - return ret; + return iov_iter_rw(iter) == READ + ? bch2_read_iter(iocb, iter) + : bch2_write_iter(iocb, iter); } #endif /* NO_BCACHEFS_FS */ diff --git a/fs/vfs/swap.h b/fs/vfs/swap.h index 50e06967c..7d5108fa7 100644 --- a/fs/vfs/swap.h +++ b/fs/vfs/swap.h @@ -4,13 +4,13 @@ #ifndef NO_BCACHEFS_FS +struct kiocb; +struct iov_iter; struct swap_info_struct; int bch2_swap_activate(struct swap_info_struct *, struct file *, sector_t *); void bch2_swap_deactivate(struct file *); int bch2_swap_rw(struct kiocb *, struct iov_iter *); -#else - #endif /* NO_BCACHEFS_FS */ #endif /* _BCACHEFS_VFS_SWAP_H */