Skip to content

Redfs ubuntu hwe writeback switch to dio #186

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
hbirth wants to merge 2 commits into from
Closed

Redfs ubuntu hwe writeback switch to dio #186

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
74812ac
fuse: take inode lock shared for DLM-serialised buffered writes
hbirth Jul 7, 2026
f6aadcc
fuse: switch to direct IO on an inode-invalidation notify storm
hbirth Jul 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
217 changes: 205 additions & 12 deletions fs/fuse/file.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -411,6 +411,18 @@ static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff,
if (likely(fi)) {
spin_lock(&fi->lock);
list_del(&ff->write_entry);
/*
* Leave forced direct IO mode once the last writer is gone: with
* no local writer left there is no cached-write contention with
* the remote modifier that triggered the switch. Restore
* FUSE_I_CACHE_IO_MODE for any frozen cached opens.
*/
if (test_bit(FUSE_I_FORCE_DIO, &fi->state) &&
list_empty(&fi->write_files)) {
clear_bit(FUSE_I_FORCE_DIO, &fi->state);
if (fi->iocachectr > 0)
set_bit(FUSE_I_CACHE_IO_MODE, &fi->state);
}
spin_unlock(&fi->lock);
}
spin_lock(&fc->lock);
Expand Down Expand Up @@ -449,9 +461,24 @@ void fuse_file_release(struct inode *inode, struct fuse_file *ff,
struct fuse_inode *fi = get_fuse_inode(inode);
struct fuse_release_args *ra = &ff->args->release_args;
int opcode = isdir ? FUSE_RELEASEDIR : FUSE_RELEASE;
bool was_force_dio = test_bit(FUSE_I_FORCE_DIO, &fi->state);

fuse_prepare_release(fi, ff, open_flags, opcode, false);

/*
* If this release dropped the last writer, fuse_prepare_release()
* cleared the forced-direct-IO latch (under fi->lock). Drop any clean
* folios a read racing the latch may have repopulated so they cannot be
* served stale once caching mode resumes. No inode lock or
* wb_inval_rwsem: release may run on the fuse server thread (async fput
* from aio completion), where blocking on a contended inode lock could
* stall the connection. Writes were routed direct while latched, so
* only clean folios exist and this invalidate is server-free; the last
* writer is gone, so no forced-dio writer can race the drop.
*/
if (was_force_dio && !test_bit(FUSE_I_FORCE_DIO, &fi->state))
invalidate_inode_pages2(inode->i_mapping);

if (ra && ff->flock) {
ra->inarg.release_flags |= FUSE_RELEASE_FLOCK_UNLOCK;
ra->inarg.lock_owner = fuse_lock_owner_id(ff->fm->fc, id);
Expand Down Expand Up @@ -1513,9 +1540,15 @@ static bool fuse_dio_wr_exclusive_lock(struct kiocb *iocb, struct iov_iter *from
struct fuse_file *ff = file->private_data;
struct inode *inode = file_inode(iocb->ki_filp);
struct fuse_inode *fi = get_fuse_inode(inode);
bool force_dio = test_bit(FUSE_I_FORCE_DIO, &fi->state);

/* Server side has to advise that it supports parallel dio writes. */
if (!(ff->open_flags & FOPEN_PARALLEL_DIRECT_WRITES))
/*
* Server side has to advise that it supports parallel dio writes.
* When the inode is latched into forced direct IO, parallel writes are
* used unconditionally: the page cache has been flushed and is bypassed
* for this inode.
*/
if (!force_dio && !(ff->open_flags & FOPEN_PARALLEL_DIRECT_WRITES))
return true;

/*
Expand All @@ -1526,7 +1559,7 @@ static bool fuse_dio_wr_exclusive_lock(struct kiocb *iocb, struct iov_iter *from
return true;

/* shared locks are not allowed with parallel page cache IO */
if (test_bit(FUSE_I_CACHE_IO_MODE, &fi->state))
if (!force_dio && test_bit(FUSE_I_CACHE_IO_MODE, &fi->state))
return true;

/* Parallel dio beyond EOF is not supported, at least for now. */
Expand All @@ -1553,9 +1586,14 @@ static void fuse_dio_lock(struct kiocb *iocb, struct iov_iter *from,
* should be performed only after taking shared inode lock.
* Previous past eof check was without inode lock and might
* have raced, so check it again.
*
* Under the forced-dio latch the cached/uncached accounting is
* bypassed (the latch guarantees the cache is flushed and not
* repopulated), so only re-check the past-eof condition.
*/
if (fuse_io_past_eof(iocb, from) ||
fuse_inode_uncached_io_start(fi, NULL) != 0) {
(!test_bit(FUSE_I_FORCE_DIO, &fi->state) &&
fuse_inode_uncached_io_start(fi, NULL) != 0)) {
inode_unlock_shared(inode);
inode_lock(inode);
*exclusive = true;
Expand All @@ -1571,8 +1609,13 @@ static void fuse_dio_unlock(struct kiocb *iocb, bool exclusive)
if (exclusive) {
inode_unlock(inode);
} else {
/* Allow opens in caching mode after last parallel dio end */
fuse_inode_uncached_io_end(fi);
/*
* Allow opens in caching mode after last parallel dio end.
* Skipped under the forced-dio latch, which never took an
* uncached_io reference in fuse_dio_lock().
*/
if (!test_bit(FUSE_I_FORCE_DIO, &fi->state))
fuse_inode_uncached_io_end(fi);
inode_unlock_shared(inode);
}
}
Expand Down Expand Up @@ -1666,6 +1709,37 @@ static ssize_t fuse_writeback_write_iter(struct kiocb *iocb,
return written < 0 ? written : total_written;
}

/*
* @return true if an exclusive lock is needed for a cached (buffered) write.
*/
static bool fuse_cache_wr_exclusive_lock(struct kiocb *iocb, bool cache_mode)
{
struct inode *inode = file_inode(iocb->ki_filp);
struct fuse_conn *fc = get_fuse_conn(inode);

/*
* Without the DLM the inode rwsem is the only cross-writer exclusion.
* Outside writeback-cache mode (cache_mode == false: no writeback cache,
* or a suid/sgid drop that needs notify_change) the write is covered by
* neither the DLM write lock nor a synchronous server round-trip, so
* keep the exclusive lock.
*/
if (!fc->dlm || !cache_mode)
return true;

/* O_DIRECT writes fall back to generic_file_direct_write(). */
if (iocb->ki_flags & IOCB_DIRECT)
return true;

/* Append needs the eventual EOF - always needs an exclusive lock. */
if (iocb->ki_flags & IOCB_APPEND)
return true;

return false;
}

static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from);

static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
{
struct file *file = iocb->ki_filp;
Expand All @@ -1675,7 +1749,21 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
struct inode *inode = mapping->host;
ssize_t err, count;
struct fuse_conn *fc = get_fuse_conn(inode);
struct fuse_inode *fi = get_fuse_inode(inode);
bool writeback = false;
bool cache_mode = false;
bool exclusive;
bool wb_guard = false;

/*
* The inode may have been latched into forced direct IO -- by a
* NOTIFY_INVAL_INODE arriving while this inode is open for writing here
* -- after this write was routed to the cached path but before it took
* any lock. Re-route to the direct path (before taking a DLM lock) so
* we do not repopulate the page cache the latch just dropped.
*/
if (fuse_inode_force_dio(inode))
return fuse_direct_write_iter(iocb, from);

if (fc->writeback_cache) {
/* Update size (EOF optimization) and mode (SUID clearing) */
Expand All @@ -1684,8 +1772,10 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
if (err)
return err;

if (!fc->handle_killpriv_v2 ||
!setattr_should_drop_suidgid(idmap, file_inode(file))) {
cache_mode = !fc->handle_killpriv_v2 ||
!setattr_should_drop_suidgid(idmap, file_inode(file));

if (cache_mode) {
writeback = true;

/*
Expand Down Expand Up @@ -1713,7 +1803,35 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
}
}

inode_lock(inode);
exclusive = fuse_cache_wr_exclusive_lock(iocb, cache_mode);
if (exclusive)
inode_lock(inode);
else
inode_lock_shared(inode);

/*
* The forced-direct-IO latch feature is active under writeback+dlm;
* hold wb_inval_rwsem for read across the page-cache dirtying so a
* concurrent NOTIFY_INVAL_INODE -- which latches the inode under the
* write side of this lock via a non-blocking trylock -- cannot strand
* the folios we are about to write. Re-check the latch under it (it may
* have been set while we blocked on the inode lock) and re-route to the
* direct path if set. Taken before task_io_account_write() so a
* re-route is not double-counted; the DLM write lock taken above is
* harmless as the direct path does its own server coordination.
*/
wb_guard = fc->writeback_cache && fc->dlm;
if (wb_guard) {
down_read(&fi->wb_inval_rwsem);
if (fuse_inode_force_dio(inode)) {
up_read(&fi->wb_inval_rwsem);
if (exclusive)
inode_unlock(inode);
else
inode_unlock_shared(inode);
return fuse_direct_write_iter(iocb, from);
}
}

err = count = generic_write_checks(iocb, from);
if (err <= 0)
Expand All @@ -1732,7 +1850,51 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
written = direct_write_fallback(iocb, from, written,
fuse_perform_write(iocb, from));
} else if (writeback) {
loff_t pos = iocb->ki_pos;
loff_t end = pos + count;
loff_t orig_size = 0;
bool extended = false;

if (fc->dlm && !exclusive && end > i_size_read(inode)) {
/*
* Lockless pre-check above keeps in-bounds writes off
* fi->lock; under the shared inode lock i_size only grows
* (extenders take fi->lock, truncate is excluded), so a
* stale read can only over-trigger this slow path, never
* miss an extension. Re-check authoritatively here.
*/
spin_lock(&fi->lock);
orig_size = i_size_read(inode);
if (end > orig_size) {
i_size_write(inode, end);
extended = true;
}
spin_unlock(&fi->lock);

/* Zero the tail of the folio straddling the old EOF. */
if (extended && orig_size < pos)
pagecache_isize_extended(inode, orig_size, pos);
}

written = fuse_writeback_write_iter(iocb, from, file);

/*
* Reconcile the speculative extension with what was actually
* written. Only retract the tail if no concurrent extender has
* pushed i_size past our claim; otherwise [reached, end) is a
* legitimate hole inside their extension and must remain.
*/
if (extended) {
loff_t reached = written > 0 ? pos + written : orig_size;

if (reached < end) {
spin_lock(&fi->lock);
if (i_size_read(inode) == end)
i_size_write(inode, reached);
spin_unlock(&fi->lock);
}
}

if (written < 0) {
err = written;
goto out;
Expand All @@ -1741,7 +1903,12 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
written = fuse_perform_write(iocb, from);
}
out:
inode_unlock(inode);
if (wb_guard)
up_read(&fi->wb_inval_rwsem);
if (exclusive)
inode_unlock(inode);
else
inode_unlock_shared(inode);
if (written > 0)
written = generic_write_sync(iocb, written);

Expand Down Expand Up @@ -2042,7 +2209,7 @@ static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
return fuse_dax_read_iter(iocb, to);

/* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */
if (ff->open_flags & FOPEN_DIRECT_IO)
if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_inode_force_dio(inode))
return fuse_direct_read_iter(iocb, to);
else if (fuse_file_passthrough(ff))
return fuse_passthrough_read_iter(iocb, to);
Expand All @@ -2063,7 +2230,7 @@ static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
return fuse_dax_write_iter(iocb, from);

/* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */
if (ff->open_flags & FOPEN_DIRECT_IO)
if ((ff->open_flags & FOPEN_DIRECT_IO) || fuse_inode_force_dio(inode))
return fuse_direct_write_iter(iocb, from);
else if (fuse_file_passthrough(ff))
return fuse_passthrough_write_iter(iocb, from);
Expand Down Expand Up @@ -2691,6 +2858,29 @@ static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma)
else if (fuse_inode_backing(get_fuse_inode(inode)))
return -ENODEV;

/*
* If the inode was latched into forced direct IO after a remote-modify
* notification, a mapping needs the page cache, so revert to caching
* mode. Revert without the inode lock or wb_inval_rwsem: ->mmap runs
* under mmap_lock and the buffered write path holds both across a fault
* on the user buffer (which takes mmap_lock), so taking either here
* would invert lock order (ABBA). Clearing the latch and dropping the
* cache is sufficient -- writers re-check the latch and route to cached
* IO once it is clear, and in-flight parallel dio drains itself. Cached
* opens frozen while latched are still counted in iocachectr, so restore
* FUSE_I_CACHE_IO_MODE for them.
*/
if (fuse_inode_force_dio(inode)) {
struct fuse_inode *fi = get_fuse_inode(inode);

spin_lock(&fi->lock);
clear_bit(FUSE_I_FORCE_DIO, &fi->state);
if (fi->iocachectr > 0)
set_bit(FUSE_I_CACHE_IO_MODE, &fi->state);
spin_unlock(&fi->lock);
invalidate_inode_pages2(file->f_mapping);
}

/*
* FOPEN_DIRECT_IO handling is special compared to O_DIRECT,
* as does not allow MAP_SHARED mmap without FUSE_DIRECT_IO_ALLOW_MMAP.
Expand Down Expand Up @@ -3502,6 +3692,9 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags)
fi->iocachectr = 0;
init_waitqueue_head(&fi->page_waitq);
init_waitqueue_head(&fi->direct_io_waitq);
init_rwsem(&fi->wb_inval_rwsem);
fi->notify_stamp = jiffies;
fi->notify_interval_ewma = FUSE_NOTIFY_EWMA_SEED << FUSE_NOTIFY_EWMA_SHIFT;

if (IS_ENABLED(CONFIG_FUSE_DAX))
fuse_dax_inode_init(inode, flags);
Expand Down
Loading