fuse: fence cached reads against NOTIFY invalidate with a percpu gate

fs/fuse/file.c

@@ -1224,11 +1224,16 @@ static void fuse_readahead(struct readahead_control *rac)
 	}
 }
 
+static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to);
+
 static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to)
 {
 	struct file *file = iocb->ki_filp;
 	struct inode *inode = file->f_mapping->host;
 	struct fuse_conn *fc = get_fuse_conn(inode);
+	struct fuse_inode *fi = get_fuse_inode(inode);
+	struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem;
+	ssize_t res;
 
 	/*
 	 * In auto invalidate mode, always update attributes on read.
@@ -1249,7 +1254,29 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to)
 		fuse_get_dlm_lock(file, iocb->ki_pos,
 				  iov_iter_count(to), FUSE_PAGE_LOCK_READ);
 
-	return generic_file_read_iter(iocb, to);
+	/*
+	 * Fence the cache-serving read against a NOTIFY invalidate so we never
+	 * hand back a folio the server has just superseded.  The gate read side
+	 * is per-CPU cheap; the NOTIFY holds the write side with priority.
+	 * Re-check the forced-DIO latch under it: if a storm latched us while we
+	 * waited on a pending writer, reroute to direct like the buffered write
+	 * path, so we do not repopulate the cache the latch just dropped.
+	 * wb_sem is NULL on non-writeback+dlm mounts (gate inactive).
+	 */
+	if (wb_sem) {
+		percpu_down_read(wb_sem);
+		if (fuse_inode_force_dio(inode)) {
+			percpu_up_read(wb_sem);
+			return fuse_direct_read_iter(iocb, to);
+		}
+	}
+
+	res = generic_file_read_iter(iocb, to);
+
+	if (wb_sem)
+		percpu_up_read(wb_sem);
+
+	return res;
 }
 
 static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff,
@@ -1747,6 +1774,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	ssize_t err, count;
 	struct fuse_conn *fc = get_fuse_conn(inode);
 	struct fuse_inode *fi = get_fuse_inode(inode);
+	struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem;
 	bool writeback = false;
 	bool wb_guard = false;
 	bool exclusive = true;
@@ -1806,20 +1834,21 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
 
 	/*
 	 * The forced-direct-IO latch feature is active under writeback+dlm;
-	 * hold wb_inval_rwsem for read across the page-cache dirtying so a
-	 * concurrent NOTIFY_INVAL_INODE -- which latches the inode under the
-	 * write side of this lock via a non-blocking trylock -- cannot strand
-	 * the folios we are about to write.  Re-check the latch under it (it may
-	 * have been set while we blocked on the inode lock) and re-route to the
-	 * direct path if set.  Taken before task_io_account_write() so a
-	 * re-route is not double-counted; the DLM write lock taken above is
-	 * harmless as the direct path does its own server coordination.
+	 * hold the coherency gate (wb_inval_rwsem) for read across the
+	 * page-cache dirtying so a concurrent NOTIFY_INVAL_INODE -- which takes
+	 * the write side (blocking, with priority) around its invalidate + latch
+	 * set -- cannot strand the folios we are about to write.  Re-check the
+	 * latch under it (it may have been set while we blocked on the inode
+	 * lock) and re-route to the direct path if set.  Taken before
+	 * task_io_account_write() so a re-route is not double-counted; the DLM
+	 * write lock taken above is harmless as the direct path does its own
+	 * server coordination.
 	 */
-	wb_guard = fc->writeback_cache && fc->dlm;
+	wb_guard = !!wb_sem;
 	if (wb_guard) {
-		down_read(&fi->wb_inval_rwsem);
+		percpu_down_read(wb_sem);
 		if (fuse_inode_force_dio(inode)) {
-			up_read(&fi->wb_inval_rwsem);
+			percpu_up_read(wb_sem);
 			fuse_cache_wr_unlock(inode, exclusive);
 			return fuse_direct_write_iter(iocb, from);
 		}
@@ -1911,7 +1940,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	}
 out:
 	if (wb_guard)
-		up_read(&fi->wb_inval_rwsem);
+		percpu_up_read(wb_sem);
 	fuse_cache_wr_unlock(inode, exclusive);
 	if (written > 0)
 		written = generic_write_sync(iocb, written);
@@ -3705,7 +3734,23 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags)
 	fi->iocachectr = 0;
 	init_waitqueue_head(&fi->page_waitq);
 	init_waitqueue_head(&fi->direct_io_waitq);
-	init_rwsem(&fi->wb_inval_rwsem);
+	/*
+	 * Coherency gate for the forced-direct-IO feature; only writeback+dlm
+	 * regular files need it.  A percpu_rw_semaphore embeds per-CPU state,
+	 * so allocate it out of line and only when the mount can use it rather
+	 * than paying it on every inode.  On failure leave it NULL: the gate
+	 * stays inactive (best-effort invalidate) and the inode is still usable.
+	 */
+	fi->wb_inval_rwsem = NULL;
+	if (fc->writeback_cache && fc->dlm) {
+		struct percpu_rw_semaphore *sem = kmalloc(sizeof(*sem), GFP_KERNEL);
+
+		if (sem && percpu_init_rwsem(sem)) {
+			kfree(sem);
+			sem = NULL;
+		}
+		fi->wb_inval_rwsem = sem;
+	}
 	fi->notify_stamp = jiffies;
 	fi->notify_interval_ewma = FUSE_NOTIFY_EWMA_SEED << FUSE_NOTIFY_EWMA_SHIFT;
 

fs/fuse/fuse_i.h

@@ -23,6 +23,7 @@
 #include <linux/backing-dev.h>
 #include <linux/mutex.h>
 #include <linux/rwsem.h>
+#include <linux/percpu-rwsem.h>
 #include <linux/rbtree.h>
 #include <linux/poll.h>
 #include <linux/workqueue.h>
@@ -202,19 +203,29 @@ struct fuse_inode {
 			struct fuse_dlm_cache dlm_locked_areas;
 
 			/*
-			 * Serializes buffered-write page-cache dirtying against
-			 * the forced-direct-IO latch transition driven by
-			 * NOTIFY_INVAL_INODE (fuse_reverse_inval_inode()), which
-			 * may be delivered by the same server thread that still
-			 * owes a reply to an in-flight write holding the inode
-			 * lock.  The buffered writer holds this for read around
-			 * the dirtying and re-checks the latch under it; the
-			 * NOTIFY latch site takes it for write (trylock, never
-			 * blocking) around its page-cache invalidate + latch set.
-			 * Only regular files initialise it -- it shares storage
-			 * with the readdir-cache union arm.
+			 * Per-inode read/write coherency gate for the
+			 * forced-direct-IO feature.  Cache-serving buffered reads
+			 * and buffered writes hold it for read; being a
+			 * percpu_rw_semaphore the read side is per-CPU cheap and
+			 * scales on a shared file.  The NOTIFY invalidate
+			 * (fuse_reverse_inval_inode()) holds it for write, which
+			 * BLOCKS so the coherency notify has priority: it fences
+			 * cache-serving reads (and buffered writes) out for the
+			 * whole invalidate, so no folio a remote modify has
+			 * superseded is ever handed back.
+			 *
+			 * The write side may run on the server thread delivering
+			 * the notify, so a blocking writer is safe only under a
+			 * server that services request replies on threads other
+			 * than the one delivering the notify (see the NOTIFY site).
+			 *
+			 * Allocated out of line only for writeback+dlm regular
+			 * files (it shares storage with the readdir-cache union
+			 * arm); NULL on other mounts and on allocation failure,
+			 * where the gate is inactive and the invalidate falls back
+			 * to best-effort.
 			 */
-			struct rw_semaphore wb_inval_rwsem;
+			struct percpu_rw_semaphore *wb_inval_rwsem;
 
 			/*
 			 * Rate of FUSE_NOTIFY_INVAL_INODE data invalidations

fs/fuse/inode.c

@@ -207,6 +207,23 @@ static void fuse_evict_inode(struct inode *inode)
 		WARN_ON(!list_empty(&fi->queued_writes));
 		fuse_dlm_cache_release_locks(fi);
 	}
+
+	/*
+	 * Free the coherency gate here rather than in ->free_inode: that runs
+	 * from an RCU callback, where percpu_free_rwsem() may sleep in
+	 * rcu_sync_dtor() if the write side has not fully quiesced.  No user
+	 * can remain by eviction time: gate readers hold a file reference and
+	 * a concurrent notify holds an inode reference.  wb_inval_rwsem lives
+	 * in the regular-file union arm and is only ever allocated for regular
+	 * files, so gate on S_ISREG (but not fuse_is_bad() -- bad-marked
+	 * regular files still own a gate); a directory's overlapping
+	 * readdir-cache fields must not be misread.
+	 */
+	if (S_ISREG(inode->i_mode) && fi->wb_inval_rwsem) {
+		percpu_free_rwsem(fi->wb_inval_rwsem);
+		kfree(fi->wb_inval_rwsem);
+		fi->wb_inval_rwsem = NULL;
+	}
 }
 
 static int fuse_reconfigure(struct fs_context *fsc)
@@ -789,6 +806,7 @@ static bool fuse_notify_inval_hot(struct fuse_inode *fi)
 int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid,
 			     loff_t offset, loff_t len)
 {
+	struct percpu_rw_semaphore *wb_sem = NULL;
 	struct fuse_inode *fi;
 	struct inode *inode;
 	pgoff_t pg_start;
@@ -831,73 +849,92 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid,
 			 * Note that this can lead to some inconsistencies if
 			 * the fuse server sends unaligned data */
 			fuse_dlm_unlock_range(fi,
-								offset,
-								pg_end == -1 ? 0 :
-								(offset + len - 1));
+						offset,
+						pg_end == -1 ? 0 :
+						(offset + len - 1));
 
 		/*
 		 * A data invalidation means another (remote) entity is modifying
-		 * the file.  Keep a moving average of how fast these notifications
-		 * arrive for the whole inode; when they come in a rapid stream --
-		 * a remote writer repeatedly invalidating the file -- and it is
-		 * also open for writing here, latch the inode into direct IO:
-		 * reads and writes are served direct (from the server) until the
-		 * last writer closes or the inode is mmapped.  A lone or occasional
-		 * notify keeps the average high and does not trip the switch.  Only
-		 * under writeback+dlm, where the buffered write's RMW read is
-		 * skipped so its wb_inval_rwsem read-side section is free of server
-		 * round-trips.
+		 * the file.  Two things happen here:
+		 *
+		 * 1. Coherency.  Drop the affected page-cache range so no local
+		 *    read returns a folio the remote modify has superseded.  This
+		 *    runs under the write side of the per-inode coherency gate
+		 *    (wb_inval_rwsem), which fences cache-serving buffered reads
+		 *    and buffered writes out for the whole invalidate.  Unlike the
+		 *    old best-effort trylock this BLOCKS -- the notify has
+		 *    priority: percpu_down_write() parks new gate readers, drains
+		 *    in-flight ones, then invalidates.  A blocking writer here is
+		 *    safe only under a server that services request replies on
+		 *    threads other than the one delivering this notify: the write
+		 *    side waits for gate readers to drain, and a cache-miss read
+		 *    holds the read side across its FUSE_READ round-trip.  redfs'
+		 *    dlm server provides that contract; a server that cannot must
+		 *    not enable writeback+dlm.
 		 *
-		 * fuse_notify_inval_hot() updates the average under fi->lock and is
-		 * called for every data invalidation so it stays current even while
-		 * no local writer is open.  When it trips, take wb_inval_rwsem for
-		 * write so the buffered write path -- which holds it for read across
-		 * its dirtying and re-checks the latch under it -- cannot strand
-		 * dirty folios after the cache is dropped.  Use a trylock and never
-		 * block: this may run on the server thread that still owes an
-		 * in-flight write (holding the inode lock) its reply, so blocking on
-		 * the rwsem or the inode lock would deadlock.  If the writer has
-		 * gone, skip the latch this round (best effort); the invalidate
-		 * still runs.  Only regular files initialise the average and the
-		 * rwsem (they share storage with the readdir-cache union arm), so
-		 * gate on S_ISREG.  When latched, drop the whole mapping rather than
-		 * just the notified range, or dirty folios outside it would be
-		 * invisible to the forced direct reads (stale read / lost write).
+		 * 2. Latch.  Keep a moving average (fuse_notify_inval_hot(), under
+		 *    fi->lock, updated for every data invalidation) of how fast
+		 *    these arrive; when they come in a rapid stream -- a remote
+		 *    writer repeatedly invalidating -- and the inode is also open
+		 *    for writing here, latch it into direct IO until the last
+		 *    writer closes or it is mmapped.  When latched, drop the whole
+		 *    mapping rather than just the notified range, or dirty folios
+		 *    outside it would be invisible to the forced direct reads
+		 *    (stale read / lost write).
+		 *
+		 * The gate (and the average) exist only for writeback+dlm regular
+		 * files, and not while mmapped; elsewhere wb_sem is NULL and the
+		 * invalidate runs unserialized (best-effort), as before.
 		 */
-		if (S_ISREG(inode->i_mode) && fc->writeback_cache && fc->dlm &&
-		    !FUSE_IS_DAX(inode) && !fuse_inode_backing(fi) &&
-		    !mapping_mapped(inode->i_mapping)) {
-			bool hot, has_writer;
+		if (S_ISREG(inode->i_mode) && fc->writeback_cache &&
+		    fc->dlm && !FUSE_IS_DAX(inode) &&
+		    !fuse_inode_backing(fi) &&
+		    !mapping_mapped(inode->i_mapping))
+			wb_sem = fi->wb_inval_rwsem;
+
+		if (wb_sem) {
+			bool hot, has_writer, latched = false;
 
 			spin_lock(&fi->lock);
 			hot = fuse_notify_inval_hot(fi);
 			has_writer = !list_empty(&fi->write_files);
 			spin_unlock(&fi->lock);
 
-			if (hot && has_writer && !fuse_inode_force_dio(inode) &&
-			    down_write_trylock(&fi->wb_inval_rwsem)) {
-				bool latched = false;
+			/*
+			 * Priority write side: park new gate readers,
+			 * drain in-flight ones, then invalidate.  Blocks
+			 * (unlike the old trylock) -- see the contract in
+			 * the comment above.
+			 */
+			percpu_down_write(wb_sem);
 
+			if (hot && has_writer &&
+			    !fuse_inode_force_dio(inode)) {
 				spin_lock(&fi->lock);
 				if (!list_empty(&fi->write_files)) {
 					set_bit(FUSE_I_FORCE_DIO, &fi->state);
 					latched = true;
 				}
 				spin_unlock(&fi->lock);
+			}
 
-				if (latched) {
-					pr_info_ratelimited("FUSE: inode %llu latched to direct IO on invalidation notify storm\n",
-							    nodeid);
-					invalidate_inode_pages2(inode->i_mapping);
-				} else {
-					invalidate_inode_pages2_range(inode->i_mapping,
-								      pg_start, pg_end);
-				}
-				up_write(&fi->wb_inval_rwsem);
-			} else {
+			/*
+			 * Latched: drop the whole mapping (dirty folios
+			 * outside the notified range would be invisible to
+			 * the forced direct reads).  Otherwise just the
+			 * notified range.
+			 */
+			if (fuse_inode_force_dio(inode))
+				invalidate_inode_pages2(inode->i_mapping);
+			else
 				invalidate_inode_pages2_range(inode->i_mapping,
 							      pg_start, pg_end);
-			}
+
+			percpu_up_write(wb_sem);
+
+			if (latched)
+				pr_info_ratelimited("FUSE: inode %llu latched to direct IO on invalidation notify storm\n",
+						    nodeid);
 		} else {
 			invalidate_inode_pages2_range(inode->i_mapping,
 						      pg_start, pg_end);