Messages in this thread Patch in this message | | | From | Nikolay Borisov <> | Subject | [PATCH] iov_iter: Microoptimize fault_in_iov_iter_(writable|readable) | Date | Fri, 22 Jul 2022 16:19:32 +0300 |
| |
Inverting the condition inside the 2 functions results in reduced nesting as well as saving 20 bytes of generated codeu as well as eliminating a mandatory branch when a proper userspace iov iter is passed:
add/remove: 0/0 grow/shrink: 0/2 up/down: 0/-20 (-20) Function old new delta fault_in_iov_iter_writeable 131 121 -10 fault_in_iov_iter_readable 131 121 -10
That's because in the original version the compiler duplicates the function epilogue right after the initial branch, resulting in the following code:
0x000000000000081c <+12>: je 0x828 <fault_in_iov_iter_readable+24> 0x000000000000081e <+14>: pop %rbx 0x000000000000081f <+15>: mov %r12,%rax 0x0000000000000822 <+18>: pop %rbp 0x0000000000000823 <+19>: pop %r12 0x0000000000000825 <+21>: pop %r13 0x0000000000000827 <+23>: retq 0x0000000000000828 <+24>: cmp %rsi,0x10(%rdi) ;loop begins
There is always a branch penalty before actual executioni begins, whilst with this patch the generated code is:
0x00000000000004ac <+12>: jne 0x507 <fault_in_iov_iter_readable+103> 0x00000000000004ae <+14>: cmp %rsi,0x10(%rdi)
The branch is taken only when the passed in iter is not an iovec
Signed-off-by: Nikolay Borisov <nborisov@suse.com> --- lib/iov_iter.c | 76 +++++++++++++++++++++++++------------------------- 1 file changed, 38 insertions(+), 38 deletions(-)
diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 0b64695ab632..897148d9cd12 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -443,26 +443,26 @@ static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t by */ size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t size) { - if (iter_is_iovec(i)) { - size_t count = min(size, iov_iter_count(i)); - const struct iovec *p; - size_t skip; - - size -= count; - for (p = i->iov, skip = i->iov_offset; count; p++, skip = 0) { - size_t len = min(count, p->iov_len - skip); - size_t ret; - - if (unlikely(!len)) - continue; - ret = fault_in_readable(p->iov_base + skip, len); - count -= len - ret; - if (ret) - break; - } - return count + size; + size_t count = min(size, iov_iter_count(i)); + const struct iovec *p; + size_t skip; + + if (!iter_is_iovec(i)) + return 0; + + size -= count; + for (p = i->iov, skip = i->iov_offset; count; p++, skip = 0) { + size_t len = min(count, p->iov_len - skip); + size_t ret; + + if (unlikely(!len)) + continue; + ret = fault_in_readable(p->iov_base + skip, len); + count -= len - ret; // reduce count by the number faulted in + if (ret) + break; } - return 0; + return count + size; } EXPORT_SYMBOL(fault_in_iov_iter_readable);
@@ -482,26 +482,26 @@ EXPORT_SYMBOL(fault_in_iov_iter_readable); */ size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t size) { - if (iter_is_iovec(i)) { - size_t count = min(size, iov_iter_count(i)); - const struct iovec *p; - size_t skip; - - size -= count; - for (p = i->iov, skip = i->iov_offset; count; p++, skip = 0) { - size_t len = min(count, p->iov_len - skip); - size_t ret; - - if (unlikely(!len)) - continue; - ret = fault_in_safe_writeable(p->iov_base + skip, len); - count -= len - ret; - if (ret) - break; - } - return count + size; + size_t count = min(size, iov_iter_count(i)); + const struct iovec *p; + size_t skip; + + if (!iter_is_iovec(i)) + return 0; + + size -= count; + for (p = i->iov, skip = i->iov_offset; count; p++, skip = 0) { + size_t len = min(count, p->iov_len - skip); + size_t ret; + + if (unlikely(!len)) + continue; + ret = fault_in_safe_writeable(p->iov_base + skip, len); + count -= len - ret; + if (ret) + break; } - return 0; + return count + size; } EXPORT_SYMBOL(fault_in_iov_iter_writeable);
-- 2.25.1
| |