diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 42e3c37edb6947..9c8d180e3f8cbd 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1224,11 +1224,16 @@ static void fuse_readahead(struct readahead_control *rac) } } +static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to); + static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) { struct file *file = iocb->ki_filp; struct inode *inode = file->f_mapping->host; struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; + ssize_t res; /* * In auto invalidate mode, always update attributes on read. @@ -1249,7 +1254,29 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) fuse_get_dlm_lock(file, iocb->ki_pos, iov_iter_count(to), FUSE_PAGE_LOCK_READ); - return generic_file_read_iter(iocb, to); + /* + * Fence the cache-serving read against a NOTIFY invalidate so we never + * hand back a folio the server has just superseded. The gate read side + * is per-CPU cheap; the NOTIFY holds the write side with priority. + * Re-check the forced-DIO latch under it: if a storm latched us while we + * waited on a pending writer, reroute to direct like the buffered write + * path, so we do not repopulate the cache the latch just dropped. + * wb_sem is NULL on non-writeback+dlm mounts (gate inactive). + */ + if (wb_sem) { + percpu_down_read(wb_sem); + if (fuse_inode_force_dio(inode)) { + percpu_up_read(wb_sem); + return fuse_direct_read_iter(iocb, to); + } + } + + res = generic_file_read_iter(iocb, to); + + if (wb_sem) + percpu_up_read(wb_sem); + + return res; } static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff, @@ -1747,6 +1774,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) ssize_t err, count; struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); + struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; bool writeback = false; bool wb_guard = false; bool exclusive = true; @@ -1806,20 +1834,21 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) /* * The forced-direct-IO latch feature is active under writeback+dlm; - * hold wb_inval_rwsem for read across the page-cache dirtying so a - * concurrent NOTIFY_INVAL_INODE -- which latches the inode under the - * write side of this lock via a non-blocking trylock -- cannot strand - * the folios we are about to write. Re-check the latch under it (it may - * have been set while we blocked on the inode lock) and re-route to the - * direct path if set. Taken before task_io_account_write() so a - * re-route is not double-counted; the DLM write lock taken above is - * harmless as the direct path does its own server coordination. + * hold the coherency gate (wb_inval_rwsem) for read across the + * page-cache dirtying so a concurrent NOTIFY_INVAL_INODE -- which takes + * the write side (blocking, with priority) around its invalidate + latch + * set -- cannot strand the folios we are about to write. Re-check the + * latch under it (it may have been set while we blocked on the inode + * lock) and re-route to the direct path if set. Taken before + * task_io_account_write() so a re-route is not double-counted; the DLM + * write lock taken above is harmless as the direct path does its own + * server coordination. */ - wb_guard = fc->writeback_cache && fc->dlm; + wb_guard = !!wb_sem; if (wb_guard) { - down_read(&fi->wb_inval_rwsem); + percpu_down_read(wb_sem); if (fuse_inode_force_dio(inode)) { - up_read(&fi->wb_inval_rwsem); + percpu_up_read(wb_sem); fuse_cache_wr_unlock(inode, exclusive); return fuse_direct_write_iter(iocb, from); } @@ -1911,7 +1940,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) } out: if (wb_guard) - up_read(&fi->wb_inval_rwsem); + percpu_up_read(wb_sem); fuse_cache_wr_unlock(inode, exclusive); if (written > 0) written = generic_write_sync(iocb, written); @@ -3705,7 +3734,23 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags) fi->iocachectr = 0; init_waitqueue_head(&fi->page_waitq); init_waitqueue_head(&fi->direct_io_waitq); - init_rwsem(&fi->wb_inval_rwsem); + /* + * Coherency gate for the forced-direct-IO feature; only writeback+dlm + * regular files need it. A percpu_rw_semaphore embeds per-CPU state, + * so allocate it out of line and only when the mount can use it rather + * than paying it on every inode. On failure leave it NULL: the gate + * stays inactive (best-effort invalidate) and the inode is still usable. + */ + fi->wb_inval_rwsem = NULL; + if (fc->writeback_cache && fc->dlm) { + struct percpu_rw_semaphore *sem = kmalloc(sizeof(*sem), GFP_KERNEL); + + if (sem && percpu_init_rwsem(sem)) { + kfree(sem); + sem = NULL; + } + fi->wb_inval_rwsem = sem; + } fi->notify_stamp = jiffies; fi->notify_interval_ewma = FUSE_NOTIFY_EWMA_SEED << FUSE_NOTIFY_EWMA_SHIFT; diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index dfa11e62cc00bd..e5b2a5751cbd48 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -202,19 +203,29 @@ struct fuse_inode { struct fuse_dlm_cache dlm_locked_areas; /* - * Serializes buffered-write page-cache dirtying against - * the forced-direct-IO latch transition driven by - * NOTIFY_INVAL_INODE (fuse_reverse_inval_inode()), which - * may be delivered by the same server thread that still - * owes a reply to an in-flight write holding the inode - * lock. The buffered writer holds this for read around - * the dirtying and re-checks the latch under it; the - * NOTIFY latch site takes it for write (trylock, never - * blocking) around its page-cache invalidate + latch set. - * Only regular files initialise it -- it shares storage - * with the readdir-cache union arm. + * Per-inode read/write coherency gate for the + * forced-direct-IO feature. Cache-serving buffered reads + * and buffered writes hold it for read; being a + * percpu_rw_semaphore the read side is per-CPU cheap and + * scales on a shared file. The NOTIFY invalidate + * (fuse_reverse_inval_inode()) holds it for write, which + * BLOCKS so the coherency notify has priority: it fences + * cache-serving reads (and buffered writes) out for the + * whole invalidate, so no folio a remote modify has + * superseded is ever handed back. + * + * The write side may run on the server thread delivering + * the notify, so a blocking writer is safe only under a + * server that services request replies on threads other + * than the one delivering the notify (see the NOTIFY site). + * + * Allocated out of line only for writeback+dlm regular + * files (it shares storage with the readdir-cache union + * arm); NULL on other mounts and on allocation failure, + * where the gate is inactive and the invalidate falls back + * to best-effort. */ - struct rw_semaphore wb_inval_rwsem; + struct percpu_rw_semaphore *wb_inval_rwsem; /* * Rate of FUSE_NOTIFY_INVAL_INODE data invalidations diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index e3703f35079676..a8d443d989daae 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -207,6 +207,23 @@ static void fuse_evict_inode(struct inode *inode) WARN_ON(!list_empty(&fi->queued_writes)); fuse_dlm_cache_release_locks(fi); } + + /* + * Free the coherency gate here rather than in ->free_inode: that runs + * from an RCU callback, where percpu_free_rwsem() may sleep in + * rcu_sync_dtor() if the write side has not fully quiesced. No user + * can remain by eviction time: gate readers hold a file reference and + * a concurrent notify holds an inode reference. wb_inval_rwsem lives + * in the regular-file union arm and is only ever allocated for regular + * files, so gate on S_ISREG (but not fuse_is_bad() -- bad-marked + * regular files still own a gate); a directory's overlapping + * readdir-cache fields must not be misread. + */ + if (S_ISREG(inode->i_mode) && fi->wb_inval_rwsem) { + percpu_free_rwsem(fi->wb_inval_rwsem); + kfree(fi->wb_inval_rwsem); + fi->wb_inval_rwsem = NULL; + } } static int fuse_reconfigure(struct fs_context *fsc) @@ -789,6 +806,7 @@ static bool fuse_notify_inval_hot(struct fuse_inode *fi) int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, loff_t offset, loff_t len) { + struct percpu_rw_semaphore *wb_sem = NULL; struct fuse_inode *fi; struct inode *inode; pgoff_t pg_start; @@ -831,73 +849,92 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * Note that this can lead to some inconsistencies if * the fuse server sends unaligned data */ fuse_dlm_unlock_range(fi, - offset, - pg_end == -1 ? 0 : - (offset + len - 1)); + offset, + pg_end == -1 ? 0 : + (offset + len - 1)); /* * A data invalidation means another (remote) entity is modifying - * the file. Keep a moving average of how fast these notifications - * arrive for the whole inode; when they come in a rapid stream -- - * a remote writer repeatedly invalidating the file -- and it is - * also open for writing here, latch the inode into direct IO: - * reads and writes are served direct (from the server) until the - * last writer closes or the inode is mmapped. A lone or occasional - * notify keeps the average high and does not trip the switch. Only - * under writeback+dlm, where the buffered write's RMW read is - * skipped so its wb_inval_rwsem read-side section is free of server - * round-trips. + * the file. Two things happen here: + * + * 1. Coherency. Drop the affected page-cache range so no local + * read returns a folio the remote modify has superseded. This + * runs under the write side of the per-inode coherency gate + * (wb_inval_rwsem), which fences cache-serving buffered reads + * and buffered writes out for the whole invalidate. Unlike the + * old best-effort trylock this BLOCKS -- the notify has + * priority: percpu_down_write() parks new gate readers, drains + * in-flight ones, then invalidates. A blocking writer here is + * safe only under a server that services request replies on + * threads other than the one delivering this notify: the write + * side waits for gate readers to drain, and a cache-miss read + * holds the read side across its FUSE_READ round-trip. redfs' + * dlm server provides that contract; a server that cannot must + * not enable writeback+dlm. * - * fuse_notify_inval_hot() updates the average under fi->lock and is - * called for every data invalidation so it stays current even while - * no local writer is open. When it trips, take wb_inval_rwsem for - * write so the buffered write path -- which holds it for read across - * its dirtying and re-checks the latch under it -- cannot strand - * dirty folios after the cache is dropped. Use a trylock and never - * block: this may run on the server thread that still owes an - * in-flight write (holding the inode lock) its reply, so blocking on - * the rwsem or the inode lock would deadlock. If the writer has - * gone, skip the latch this round (best effort); the invalidate - * still runs. Only regular files initialise the average and the - * rwsem (they share storage with the readdir-cache union arm), so - * gate on S_ISREG. When latched, drop the whole mapping rather than - * just the notified range, or dirty folios outside it would be - * invisible to the forced direct reads (stale read / lost write). + * 2. Latch. Keep a moving average (fuse_notify_inval_hot(), under + * fi->lock, updated for every data invalidation) of how fast + * these arrive; when they come in a rapid stream -- a remote + * writer repeatedly invalidating -- and the inode is also open + * for writing here, latch it into direct IO until the last + * writer closes or it is mmapped. When latched, drop the whole + * mapping rather than just the notified range, or dirty folios + * outside it would be invisible to the forced direct reads + * (stale read / lost write). + * + * The gate (and the average) exist only for writeback+dlm regular + * files, and not while mmapped; elsewhere wb_sem is NULL and the + * invalidate runs unserialized (best-effort), as before. */ - if (S_ISREG(inode->i_mode) && fc->writeback_cache && fc->dlm && - !FUSE_IS_DAX(inode) && !fuse_inode_backing(fi) && - !mapping_mapped(inode->i_mapping)) { - bool hot, has_writer; + if (S_ISREG(inode->i_mode) && fc->writeback_cache && + fc->dlm && !FUSE_IS_DAX(inode) && + !fuse_inode_backing(fi) && + !mapping_mapped(inode->i_mapping)) + wb_sem = fi->wb_inval_rwsem; + + if (wb_sem) { + bool hot, has_writer, latched = false; spin_lock(&fi->lock); hot = fuse_notify_inval_hot(fi); has_writer = !list_empty(&fi->write_files); spin_unlock(&fi->lock); - if (hot && has_writer && !fuse_inode_force_dio(inode) && - down_write_trylock(&fi->wb_inval_rwsem)) { - bool latched = false; + /* + * Priority write side: park new gate readers, + * drain in-flight ones, then invalidate. Blocks + * (unlike the old trylock) -- see the contract in + * the comment above. + */ + percpu_down_write(wb_sem); + if (hot && has_writer && + !fuse_inode_force_dio(inode)) { spin_lock(&fi->lock); if (!list_empty(&fi->write_files)) { set_bit(FUSE_I_FORCE_DIO, &fi->state); latched = true; } spin_unlock(&fi->lock); + } - if (latched) { - pr_info_ratelimited("FUSE: inode %llu latched to direct IO on invalidation notify storm\n", - nodeid); - invalidate_inode_pages2(inode->i_mapping); - } else { - invalidate_inode_pages2_range(inode->i_mapping, - pg_start, pg_end); - } - up_write(&fi->wb_inval_rwsem); - } else { + /* + * Latched: drop the whole mapping (dirty folios + * outside the notified range would be invisible to + * the forced direct reads). Otherwise just the + * notified range. + */ + if (fuse_inode_force_dio(inode)) + invalidate_inode_pages2(inode->i_mapping); + else invalidate_inode_pages2_range(inode->i_mapping, pg_start, pg_end); - } + + percpu_up_write(wb_sem); + + if (latched) + pr_info_ratelimited("FUSE: inode %llu latched to direct IO on invalidation notify storm\n", + nodeid); } else { invalidate_inode_pages2_range(inode->i_mapping, pg_start, pg_end);