Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301// SPDX-License-Identifier: GPL-2.0/* * I/O and data path helper functionality. * * Borrowed from NFS Copyright (c) 2016 Trond Myklebust */
#include <linux/kernel.h>#include <linux/netfs.h>#include "internal.h"
struct netfs_wb_waiter { struct list_head link; /* Link in ictx->wb_queue */ struct task_struct *waiter; /* Waiter task; cleared when lock granted */};
/* * inode_dio_wait_interruptible - wait for outstanding DIO requests to finish * @inode: inode to wait for * * Waits for all pending direct I/O requests to finish so that we can * proceed with a truncate or equivalent operation. * * Must be called under a lock that serializes taking new references * to i_dio_count, usually by inode->i_mutex. */static int netfs_inode_dio_wait_interruptible(struct inode *inode){ if (inode_dio_finished(inode)) return 0;
inode_dio_wait_interruptible(inode); return !inode_dio_finished(inode) ? -ERESTARTSYS : 0;}
/* Call with exclusively locked inode->i_rwsem */static int netfs_block_o_direct(struct netfs_inode *ictx){ if (!test_bit(NETFS_ICTX_ODIRECT, &ictx->flags)) return 0; clear_bit(NETFS_ICTX_ODIRECT, &ictx->flags); return netfs_inode_dio_wait_interruptible(&ictx->inode);}
/** * netfs_start_io_read - declare the file is being used for buffered reads * @inode: file inode * * Declare that a buffered read operation is about to start, and ensure * that we block all direct I/O. * On exit, the function ensures that the NETFS_ICTX_ODIRECT flag is unset, * and holds a shared lock on inode->i_rwsem to ensure that the flag * cannot be changed. * In practice, this means that buffered read operations are allowed to * execute in parallel, thanks to the shared lock, whereas direct I/O * operations need to wait to grab an exclusive lock in order to set * NETFS_ICTX_ODIRECT. * Note that buffered writes and truncates both take a write lock on * inode->i_rwsem, meaning that those are serialised w.r.t. the reads. */int netfs_start_io_read(struct inode *inode) __acquires(inode->i_rwsem){ struct netfs_inode *ictx = netfs_inode(inode);
/* Be an optimist! */ if (down_read_interruptible(&inode->i_rwsem) < 0) return -ERESTARTSYS; if (test_bit(NETFS_ICTX_ODIRECT, &ictx->flags) == 0) return 0; up_read(&inode->i_rwsem);
/* Slow path.... */ if (down_write_killable(&inode->i_rwsem) < 0) return -ERESTARTSYS; if (netfs_block_o_direct(ictx) < 0) { up_write(&inode->i_rwsem); return -ERESTARTSYS; } downgrade_write(&inode->i_rwsem); return 0;}EXPORT_SYMBOL(netfs_start_io_read);
/** * netfs_end_io_read - declare that the buffered read operation is done * @inode: file inode * * Declare that a buffered read operation is done, and release the shared * lock on inode->i_rwsem. */void netfs_end_io_read(struct inode *inode) __releases(inode->i_rwsem){ up_read(&inode->i_rwsem);}EXPORT_SYMBOL(netfs_end_io_read);
/** * netfs_start_io_write - declare the file is being used for buffered writes * @inode: file inode * * Declare that a buffered read operation is about to start, and ensure * that we block all direct I/O. */int netfs_start_io_write(struct inode *inode) __acquires(inode->i_rwsem){ struct netfs_inode *ictx = netfs_inode(inode);
if (down_write_killable(&inode->i_rwsem) < 0) return -ERESTARTSYS; if (netfs_block_o_direct(ictx) < 0) { up_write(&inode->i_rwsem); return -ERESTARTSYS; } downgrade_write(&inode->i_rwsem); return 0;}EXPORT_SYMBOL(netfs_start_io_write);
/** * netfs_end_io_write - declare that the buffered write operation is done * @inode: file inode * * Declare that a buffered write operation is done, and release the * lock on inode->i_rwsem. */void netfs_end_io_write(struct inode *inode) __releases(inode->i_rwsem){ up_read(&inode->i_rwsem);}EXPORT_SYMBOL(netfs_end_io_write);
/* Call with exclusively locked inode->i_rwsem */static int netfs_block_buffered(struct inode *inode){ struct netfs_inode *ictx = netfs_inode(inode); int ret;
if (!test_bit(NETFS_ICTX_ODIRECT, &ictx->flags)) { set_bit(NETFS_ICTX_ODIRECT, &ictx->flags); if (inode->i_mapping->nrpages != 0) { unmap_mapping_range(inode->i_mapping, 0, 0, 0); ret = filemap_fdatawait(inode->i_mapping); if (ret < 0) { clear_bit(NETFS_ICTX_ODIRECT, &ictx->flags); return ret; } } } return 0;}
/** * netfs_start_io_direct - declare the file is being used for direct i/o * @inode: file inode * * Declare that a direct I/O operation is about to start, and ensure * that we block all buffered I/O. * On exit, the function ensures that the NETFS_ICTX_ODIRECT flag is set, * and holds a shared lock on inode->i_rwsem to ensure that the flag * cannot be changed. * In practice, this means that direct I/O operations are allowed to * execute in parallel, thanks to the shared lock, whereas buffered I/O * operations need to wait to grab an exclusive lock in order to clear * NETFS_ICTX_ODIRECT. * Note that buffered writes and truncates both take a write lock on * inode->i_rwsem, meaning that those are serialised w.r.t. O_DIRECT. */int netfs_start_io_direct(struct inode *inode) __acquires(inode->i_rwsem){ struct netfs_inode *ictx = netfs_inode(inode); int ret;
/* Be an optimist! */ if (down_read_interruptible(&inode->i_rwsem) < 0) return -ERESTARTSYS; if (test_bit(NETFS_ICTX_ODIRECT, &ictx->flags) != 0) return 0; up_read(&inode->i_rwsem);
/* Slow path.... */ if (down_write_killable(&inode->i_rwsem) < 0) return -ERESTARTSYS; ret = netfs_block_buffered(inode); if (ret < 0) { up_write(&inode->i_rwsem); return ret; } downgrade_write(&inode->i_rwsem); return 0;}EXPORT_SYMBOL(netfs_start_io_direct);
/** * netfs_end_io_direct - declare that the direct i/o operation is done * @inode: file inode * * Declare that a direct I/O operation is done, and release the shared * lock on inode->i_rwsem. */void netfs_end_io_direct(struct inode *inode) __releases(inode->i_rwsem){ up_read(&inode->i_rwsem);}EXPORT_SYMBOL(netfs_end_io_direct);
/* * Wait to have exclusive access to writeback. */static bool netfs_wb_begin_wait(struct netfs_inode *ictx){ struct netfs_wb_waiter waiter = {}; struct task_struct *tsk = current; bool got = false;
spin_lock(&ictx->lock);
if (test_and_set_bit_lock(NETFS_ICTX_WB_LOCK, &ictx->flags)) { get_task_struct(tsk); waiter.waiter = tsk; list_add_tail(&waiter.link, &ictx->wb_queue); } else { got = true; } spin_unlock(&ictx->lock);
if (!got) { for (;;) { set_current_state(TASK_UNINTERRUPTIBLE); /* Read waiter before accessing inode state. */ if (smp_load_acquire(&waiter.waiter) == NULL) break; schedule(); } } __set_current_state(TASK_RUNNING); return true;}
/** * netfs_wb_begin - Begin writeback, waiting if need be * @ictx: The inode to get writeback access on * @nowait: Return failure immediately rather than waiting if true * * Begin writeback to an inode, waiting for exclusive access if @nowait is * false. This prevents collection from being done out of order with respect * to the issuance of write subrequests. * * Note that writeback may be ended in a different process (e.g. the collection * function on a workqueue) than started it. * * Return: True if can proceed, false if denied. */bool netfs_wb_begin(struct netfs_inode *ictx, bool nowait){ if (!test_and_set_bit_lock(NETFS_ICTX_WB_LOCK, &ictx->flags)) return true; if (nowait) { netfs_stat(&netfs_n_wb_lock_skip); return false; } netfs_stat(&netfs_n_wb_lock_wait); return netfs_wb_begin_wait(ictx);}EXPORT_SYMBOL(netfs_wb_begin);
/* netfs_wb_end - End writeback * @ictx: The inode we have writeback access to * * End writeback access on an inode, waking up the next writeback request. */void netfs_wb_end(struct netfs_inode *ictx){ struct netfs_wb_waiter *waiter; struct task_struct *tsk;
WARN_ON_ONCE(!test_bit(NETFS_ICTX_WB_LOCK, &ictx->flags));
spin_lock(&ictx->lock);
waiter = list_first_entry_or_null(&ictx->wb_queue, struct netfs_wb_waiter, link); if (waiter) { list_del(&waiter->link); tsk = waiter->waiter; /* Write inode state before clearing waiter. */ smp_store_release(&waiter->waiter, NULL); wake_up_process(tsk); put_task_struct(tsk); } else { clear_bit_unlock(NETFS_ICTX_WB_LOCK, &ictx->flags); }
spin_unlock(&ictx->lock);}EXPORT_SYMBOL(netfs_wb_end);