Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257/* SPDX-License-Identifier: GPL-2.0-or-later *//* * Cryptographic scatter and gather helpers. * * Copyright (c) 2002 James Morris <jmorris@intercode.com.au> * Copyright (c) 2002 Adam J. Richter <adam@yggdrasil.com> * Copyright (c) 2004 Jean-Luc Cooke <jlcooke@certainkey.com> * Copyright (c) 2007 Herbert Xu <herbert@gondor.apana.org.au> */
#ifndef _CRYPTO_SCATTERWALK_H#define _CRYPTO_SCATTERWALK_H
#include <crypto/algapi.h>
#include <linux/highmem.h>#include <linux/mm.h>#include <linux/scatterlist.h>
static inline void scatterwalk_crypto_chain(struct scatterlist *head, struct scatterlist *sg, int num){ if (sg) sg_chain(head, num, sg); else sg_mark_end(head);}
static inline void scatterwalk_start(struct scatter_walk *walk, struct scatterlist *sg){ walk->sg = sg; walk->offset = sg->offset;}
/* * This is equivalent to scatterwalk_start(walk, sg) followed by * scatterwalk_skip(walk, pos). */static inline void scatterwalk_start_at_pos(struct scatter_walk *walk, struct scatterlist *sg, unsigned int pos){ while (pos > sg->length) { pos -= sg->length; sg = sg_next(sg); } walk->sg = sg; walk->offset = sg->offset + pos;}
static inline unsigned int scatterwalk_clamp(struct scatter_walk *walk, unsigned int nbytes){ unsigned int len_this_sg; unsigned int limit;
if (walk->offset >= walk->sg->offset + walk->sg->length) scatterwalk_start(walk, sg_next(walk->sg)); len_this_sg = walk->sg->offset + walk->sg->length - walk->offset;
/* * HIGHMEM case: the page may have to be mapped into memory. To avoid * the complexity of having to map multiple pages at once per sg entry, * clamp the returned length to not cross a page boundary. * * !HIGHMEM case: no mapping is needed; all pages of the sg entry are * already mapped contiguously in the kernel's direct map. For improved * performance, allow the walker to return data segments that cross a * page boundary. Do still cap the length to PAGE_SIZE, since some * users rely on that to avoid disabling preemption for too long when * using SIMD. It's also needed for when skcipher_walk uses a bounce * page due to the data not being aligned to the algorithm's alignmask. */ if (IS_ENABLED(CONFIG_HIGHMEM)) limit = PAGE_SIZE - offset_in_page(walk->offset); else limit = PAGE_SIZE;
return min3(nbytes, len_this_sg, limit);}
/* * Create a scatterlist that represents the remaining data in a walk. Uses * chaining to reference the original scatterlist, so this uses at most two * entries in @sg_out regardless of the number of entries in the original list. * Assumes that sg_init_table() was already done. */static inline void scatterwalk_get_sglist(struct scatter_walk *walk, struct scatterlist sg_out[2]){ if (walk->offset >= walk->sg->offset + walk->sg->length) scatterwalk_start(walk, sg_next(walk->sg)); sg_set_page(sg_out, sg_page(walk->sg), walk->sg->offset + walk->sg->length - walk->offset, walk->offset); scatterwalk_crypto_chain(sg_out, sg_next(walk->sg), 2);}
static inline void scatterwalk_map(struct scatter_walk *walk){ struct page *base_page = sg_page(walk->sg); unsigned int offset = walk->offset; void *addr;
if (IS_ENABLED(CONFIG_HIGHMEM)) { struct page *page;
page = base_page + (offset >> PAGE_SHIFT); offset = offset_in_page(offset); addr = kmap_local_page(page) + offset; } else { /* * When !HIGHMEM we allow the walker to return segments that * span a page boundary; see scatterwalk_clamp(). To make it * clear that in this case we're working in the linear buffer of * the whole sg entry in the kernel's direct map rather than * within the mapped buffer of a single page, compute the * address as an offset from the page_address() of the first * page of the sg entry. Either way the result is the address * in the direct map, but this makes it clearer what is really * going on. */ addr = page_address(base_page) + offset; }
walk->__addr = addr;}
/** * scatterwalk_next() - Get the next data buffer in a scatterlist walk * @walk: the scatter_walk * @total: the total number of bytes remaining, > 0 * * A virtual address for the next segment of data from the scatterlist will * be placed into @walk->addr. The caller must call scatterwalk_done_src() * or scatterwalk_done_dst() when it is done using this virtual address. * * Returns: the next number of bytes available, <= @total */static inline unsigned int scatterwalk_next(struct scatter_walk *walk, unsigned int total){ unsigned int nbytes = scatterwalk_clamp(walk, total);
scatterwalk_map(walk); return nbytes;}
static inline void scatterwalk_unmap(struct scatter_walk *walk){ if (IS_ENABLED(CONFIG_HIGHMEM)) kunmap_local(walk->__addr);}
static inline void scatterwalk_advance(struct scatter_walk *walk, unsigned int nbytes){ walk->offset += nbytes;}
/** * scatterwalk_done_src() - Finish one step of a walk of source scatterlist * @walk: the scatter_walk * @nbytes: the number of bytes processed this step, less than or equal to the * number of bytes that scatterwalk_next() returned. * * Use this if the mapped address was not written to, i.e. it is source data. */static inline void scatterwalk_done_src(struct scatter_walk *walk, unsigned int nbytes){ scatterwalk_unmap(walk); scatterwalk_advance(walk, nbytes);}
/* * Flush the dcache of any pages that overlap the region * [offset, offset + nbytes) relative to base_page. * * This should be called only when ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE, to ensure * that all relevant code (including the call to sg_page() in the caller, if * applicable) gets fully optimized out when !ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE. */static inline void __scatterwalk_flush_dcache_pages(struct page *base_page, unsigned int offset, unsigned int nbytes){ unsigned int num_pages;
base_page += offset / PAGE_SIZE; offset %= PAGE_SIZE;
/* * This is an overflow-safe version of * num_pages = DIV_ROUND_UP(offset + nbytes, PAGE_SIZE). */ num_pages = nbytes / PAGE_SIZE; num_pages += DIV_ROUND_UP(offset + (nbytes % PAGE_SIZE), PAGE_SIZE);
for (unsigned int i = 0; i < num_pages; i++) flush_dcache_page(base_page + i);}
/** * scatterwalk_done_dst() - Finish one step of a walk of destination scatterlist * @walk: the scatter_walk * @nbytes: the number of bytes processed this step, less than or equal to the * number of bytes that scatterwalk_next() returned. * * Use this if the mapped address may have been written to, i.e. it is * destination data. */static inline void scatterwalk_done_dst(struct scatter_walk *walk, unsigned int nbytes){ scatterwalk_unmap(walk); if (ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE) __scatterwalk_flush_dcache_pages(sg_page(walk->sg), walk->offset, nbytes); scatterwalk_advance(walk, nbytes);}
void scatterwalk_skip(struct scatter_walk *walk, unsigned int nbytes);
void memcpy_from_scatterwalk(void *buf, struct scatter_walk *walk, unsigned int nbytes);
void memcpy_to_scatterwalk(struct scatter_walk *walk, const void *buf, unsigned int nbytes);
void memcpy_from_sglist(void *buf, struct scatterlist *sg, unsigned int start, unsigned int nbytes);
void memcpy_to_sglist(struct scatterlist *sg, unsigned int start, const void *buf, unsigned int nbytes);
void memcpy_sglist(struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes);
/* In new code, please use memcpy_{from,to}_sglist() directly instead. */static inline void scatterwalk_map_and_copy(void *buf, struct scatterlist *sg, unsigned int start, unsigned int nbytes, int out){ if (out) memcpy_to_sglist(sg, start, buf, nbytes); else memcpy_from_sglist(buf, sg, start, nbytes);}
struct scatterlist *scatterwalk_ffwd(struct scatterlist dst[2], struct scatterlist *src, unsigned int len);
#endif /* _CRYPTO_SCATTERWALK_H */