lkml.org 
[lkml]   [2020]   [Dec]   [8]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH 2/2] block: no-copy bvec for direct IO
    Date
    The block layer spends quite a while in blkdev_direct_IO() to copy and
    initialise bio's bvec. However, if we've already got a bvec in the input
    iterator it might be reused in some cases, i.e. when new
    ITER_BVEC_FLAG_FIXED flag is set. Simple tests show considerable
    performance boost, and it also reduces memory footprint.

    Suggested-by: Matthew Wilcox <willy@infradead.org>
    [BIO_WORKINGSET] Suggested-by: Johannes Weiner <hannes@cmpxchg.org>
    Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
    ---
    fs/block_dev.c | 30 +++++++++++++++++++++++++++++-
    1 file changed, 29 insertions(+), 1 deletion(-)

    diff --git a/fs/block_dev.c b/fs/block_dev.c
    index d699f3af1a09..aee5d2e4f324 100644
    --- a/fs/block_dev.c
    +++ b/fs/block_dev.c
    @@ -349,6 +349,28 @@ static void blkdev_bio_end_io(struct bio *bio)
    }
    }

    +static int bio_iov_fixed_bvec_get_pages(struct bio *bio, struct iov_iter *iter)
    +{
    + bio->bi_vcnt = iter->nr_segs;
    + bio->bi_max_vecs = iter->nr_segs;
    + bio->bi_io_vec = (struct bio_vec *)iter->bvec;
    + bio->bi_iter.bi_bvec_done = iter->iov_offset;
    + bio->bi_iter.bi_size = iter->count;
    +
    + /*
    + * In practice groups of pages tend to be accessed/reclaimed/refaulted
    + * together. To not go over bvec for those who didn't set BIO_WORKINGSET
    + * approximate it by looking at the first page and inducing it to the
    + * whole bio
    + */
    + if (unlikely(PageWorkingset(iter->bvec->bv_page)))
    + bio_set_flag(bio, BIO_WORKINGSET);
    + bio_set_flag(bio, BIO_NO_PAGE_REF);
    +
    + iter->count = 0;
    + return 0;
    +}
    +
    static ssize_t
    __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_vecs)
    {
    @@ -368,6 +390,8 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_vecs)
    (bdev_logical_block_size(bdev) - 1))
    return -EINVAL;

    + if (iov_iter_bvec_fixed(iter))
    + nr_vecs = 0;
    bio = bio_alloc_bioset(GFP_KERNEL, nr_vecs, &blkdev_dio_pool);

    dio = container_of(bio, struct blkdev_dio, bio);
    @@ -398,7 +422,11 @@ __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_vecs)
    bio->bi_end_io = blkdev_bio_end_io;
    bio->bi_ioprio = iocb->ki_ioprio;

    - ret = bio_iov_iter_get_pages(bio, iter);
    + if (iov_iter_is_bvec(iter) && iov_iter_bvec_fixed(iter))
    + ret = bio_iov_fixed_bvec_get_pages(bio, iter);
    + else
    + ret = bio_iov_iter_get_pages(bio, iter);
    +
    if (unlikely(ret)) {
    bio->bi_status = BLK_STS_IOERR;
    bio_endio(bio);
    --
    2.24.0
    \
     
     \ /
      Last update: 2020-12-09 03:25    [W:3.147 / U:0.008 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site