Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028// SPDX-License-Identifier: GPL-2.0/* * Copyright (C) 2001 Jens Axboe <axboe@kernel.dk> */#include <linux/mm.h>#include <linux/swap.h>#include <linux/bio-integrity.h>#include <linux/blkdev.h>#include <linux/uio.h>#include <linux/iocontext.h>#include <linux/slab.h>#include <linux/init.h>#include <linux/kernel.h>#include <linux/export.h>#include <linux/mempool.h>#include <linux/workqueue.h>#include <linux/cgroup.h>#include <linux/highmem.h>#include <linux/blk-crypto.h>#include <linux/xarray.h>#include <linux/kmemleak.h>
#include <trace/events/block.h>#include "blk.h"#include "blk-rq-qos.h"#include "blk-cgroup.h"
#define ALLOC_CACHE_THRESHOLD 16#define ALLOC_CACHE_MAX 256
struct bio_alloc_cache { struct bio *free_list; struct bio *free_list_irq; unsigned int nr; unsigned int nr_irq;};
#define BIO_INLINE_VECS 4
static struct biovec_slab { int nr_vecs; char *name; struct kmem_cache *slab;} bvec_slabs[] __read_mostly = { { .nr_vecs = 16, .name = "biovec-16" }, { .nr_vecs = 64, .name = "biovec-64" }, { .nr_vecs = 128, .name = "biovec-128" }, { .nr_vecs = BIO_MAX_VECS, .name = "biovec-max" },};
static struct biovec_slab *biovec_slab(unsigned short nr_vecs){ switch (nr_vecs) { /* smaller bios use inline vecs */ case 5 ... 16: return &bvec_slabs[0]; case 17 ... 64: return &bvec_slabs[1]; case 65 ... 128: return &bvec_slabs[2]; case 129 ... BIO_MAX_VECS: return &bvec_slabs[3]; default: BUG(); return NULL; }}
/* * fs_bio_set is the bio_set containing bio and iovec memory pools used by * IO code that does not need private memory pools. */struct bio_set fs_bio_set;EXPORT_SYMBOL(fs_bio_set);
/* * Our slab pool management */struct bio_slab { struct kmem_cache *slab; unsigned int slab_ref; unsigned int slab_size; char name[12];};static DEFINE_MUTEX(bio_slab_lock);static DEFINE_XARRAY(bio_slabs);
static struct bio_slab *create_bio_slab(unsigned int size){ struct bio_slab *bslab = kzalloc_obj(*bslab);
if (!bslab) return NULL;
snprintf(bslab->name, sizeof(bslab->name), "bio-%d", size); bslab->slab = kmem_cache_create(bslab->name, size, ARCH_KMALLOC_MINALIGN, SLAB_HWCACHE_ALIGN | SLAB_TYPESAFE_BY_RCU, NULL); if (!bslab->slab) goto fail_alloc_slab;
bslab->slab_ref = 1; bslab->slab_size = size;
if (!xa_err(xa_store(&bio_slabs, size, bslab, GFP_KERNEL))) return bslab;
kmem_cache_destroy(bslab->slab);
fail_alloc_slab: kfree(bslab); return NULL;}
static inline unsigned int bs_bio_slab_size(struct bio_set *bs){ return bs->front_pad + sizeof(struct bio) + bs->back_pad;}
static inline void *bio_slab_addr(struct bio *bio){ return (void *)bio - bio->bi_pool->front_pad;}
static struct kmem_cache *bio_find_or_create_slab(struct bio_set *bs){ unsigned int size = bs_bio_slab_size(bs); struct bio_slab *bslab;
mutex_lock(&bio_slab_lock); bslab = xa_load(&bio_slabs, size); if (bslab) bslab->slab_ref++; else bslab = create_bio_slab(size); mutex_unlock(&bio_slab_lock);
if (bslab) return bslab->slab; return NULL;}
static void bio_put_slab(struct bio_set *bs){ struct bio_slab *bslab = NULL; unsigned int slab_size = bs_bio_slab_size(bs);
mutex_lock(&bio_slab_lock);
bslab = xa_load(&bio_slabs, slab_size); if (WARN(!bslab, KERN_ERR "bio: unable to find slab!\n")) goto out;
WARN_ON_ONCE(bslab->slab != bs->bio_slab);
WARN_ON(!bslab->slab_ref);
if (--bslab->slab_ref) goto out;
xa_erase(&bio_slabs, slab_size);
kmem_cache_destroy(bslab->slab); kfree(bslab);
out: mutex_unlock(&bio_slab_lock);}
/* * Make the first allocation restricted and don't dump info on allocation * failures, since we'll fall back to the mempool in case of failure. */static inline gfp_t try_alloc_gfp(gfp_t gfp){ return (gfp & ~(__GFP_DIRECT_RECLAIM | __GFP_IO)) | __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;}
void bio_uninit(struct bio *bio){#ifdef CONFIG_BLK_CGROUP if (bio->bi_blkg) { blkg_put(bio->bi_blkg); bio->bi_blkg = NULL; }#endif if (bio_integrity(bio)) bio_integrity_free(bio);
bio_crypt_free_ctx(bio);}EXPORT_SYMBOL(bio_uninit);
static void bio_free(struct bio *bio){ struct bio_set *bs = bio->bi_pool; void *p = bio;
WARN_ON_ONCE(!bs); WARN_ON_ONCE(bio->bi_max_vecs > BIO_MAX_VECS);
bio_uninit(bio); if (bio->bi_max_vecs == BIO_MAX_VECS) mempool_free(bio->bi_io_vec, &bs->bvec_pool); else if (bio->bi_max_vecs > BIO_INLINE_VECS) kmem_cache_free(biovec_slab(bio->bi_max_vecs)->slab, bio->bi_io_vec); mempool_free(p - bs->front_pad, &bs->bio_pool);}
/* * Users of this function have their own bio allocation. Subsequently, * they must remember to pair any call to bio_init() with bio_uninit() * when IO has completed, or when the bio is released. */void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table, unsigned short max_vecs, blk_opf_t opf){ bio->bi_next = NULL; bio->bi_bdev = bdev; bio->bi_opf = opf; bio->bi_flags = 0; bio->bi_ioprio = 0; bio->bi_write_hint = 0; bio->bi_write_stream = 0; bio->bi_status = 0; bio->bi_bvec_gap_bit = 0; bio->bi_iter.bi_sector = 0; bio->bi_iter.bi_size = 0; bio->bi_iter.bi_idx = 0; bio->bi_iter.bi_bvec_done = 0; bio->bi_end_io = NULL; bio->bi_private = NULL;#ifdef CONFIG_BLK_CGROUP bio->bi_blkg = NULL; bio->issue_time_ns = 0; if (bdev) bio_associate_blkg(bio);#ifdef CONFIG_BLK_CGROUP_IOCOST bio->bi_iocost_cost = 0;#endif#endif#ifdef CONFIG_BLK_INLINE_ENCRYPTION bio->bi_crypt_context = NULL;#endif#ifdef CONFIG_BLK_DEV_INTEGRITY bio->bi_integrity = NULL;#endif bio->bi_vcnt = 0;
atomic_set(&bio->__bi_remaining, 1); atomic_set(&bio->__bi_cnt, 1); bio->bi_cookie = BLK_QC_T_NONE;
bio->bi_max_vecs = max_vecs; bio->bi_io_vec = table; bio->bi_pool = NULL;}EXPORT_SYMBOL(bio_init);
/** * bio_reset - reinitialize a bio * @bio: bio to reset * @bdev: block device to use the bio for * @opf: operation and flags for bio * * Description: * After calling bio_reset(), @bio will be in the same state as a freshly * allocated bio returned bio bio_alloc_bioset() - the only fields that are * preserved are the ones that are initialized by bio_alloc_bioset(). See * comment in struct bio. */void bio_reset(struct bio *bio, struct block_device *bdev, blk_opf_t opf){ struct bio_vec *bv = bio->bi_io_vec;
bio_uninit(bio); memset(bio, 0, BIO_RESET_BYTES); atomic_set(&bio->__bi_remaining, 1); bio->bi_io_vec = bv; bio->bi_bdev = bdev; if (bio->bi_bdev) bio_associate_blkg(bio); bio->bi_opf = opf;}EXPORT_SYMBOL(bio_reset);
/** * bio_reuse - reuse a bio with the payload left intact * @bio: bio to reuse * @opf: operation and flags for the next I/O * * Allow reusing an existing bio for another operation with all set up * fields including the payload, device and end_io handler left intact. * * Typically used when @bio is first used to read data which is then written * to another location without modification. @bio must not be in-flight and * owned by the caller. Can't be used for cloned bios. * * Note: Can't be used when @bio has integrity or blk-crypto contexts for now. * Feel free to add that support when you need it, though. */void bio_reuse(struct bio *bio, blk_opf_t opf){ unsigned short vcnt = bio->bi_vcnt, i; bio_end_io_t *end_io = bio->bi_end_io; void *private = bio->bi_private;
WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)); WARN_ON_ONCE(bio_integrity(bio)); WARN_ON_ONCE(bio_has_crypt_ctx(bio));
bio_reset(bio, bio->bi_bdev, opf); for (i = 0; i < vcnt; i++) bio->bi_iter.bi_size += bio->bi_io_vec[i].bv_len; bio->bi_vcnt = vcnt; bio->bi_private = private; bio->bi_end_io = end_io;}EXPORT_SYMBOL_GPL(bio_reuse);
static struct bio *__bio_chain_endio(struct bio *bio){ struct bio *parent = bio->bi_private;
if (bio->bi_status && !parent->bi_status) parent->bi_status = bio->bi_status; bio_put(bio); return parent;}
/* * This function should only be used as a flag and must never be called. * If execution reaches here, it indicates a serious programming error. */static void bio_chain_endio(struct bio *bio){ BUG();}
/** * bio_chain - chain bio completions * @bio: the target bio * @parent: the parent bio of @bio * * The caller won't have a bi_end_io called when @bio completes - instead, * @parent's bi_end_io won't be called until both @parent and @bio have * completed; the chained bio will also be freed when it completes. * * The caller must not set bi_private or bi_end_io in @bio. */void bio_chain(struct bio *bio, struct bio *parent){ BUG_ON(bio->bi_private || bio->bi_end_io);
bio->bi_private = parent; bio->bi_end_io = bio_chain_endio; bio_inc_remaining(parent);}EXPORT_SYMBOL(bio_chain);
/** * bio_chain_and_submit - submit a bio after chaining it to another one * @prev: bio to chain and submit * @new: bio to chain to * * If @prev is non-NULL, chain it to @new and submit it. * * Return: @new. */struct bio *bio_chain_and_submit(struct bio *prev, struct bio *new){ if (prev) { bio_chain(prev, new); submit_bio(prev); } return new;}
struct bio *blk_next_bio(struct bio *bio, struct block_device *bdev, unsigned int nr_pages, blk_opf_t opf, gfp_t gfp){ return bio_chain_and_submit(bio, bio_alloc(bdev, nr_pages, opf, gfp));}EXPORT_SYMBOL_GPL(blk_next_bio);
static void bio_alloc_rescue(struct work_struct *work){ struct bio_set *bs = container_of(work, struct bio_set, rescue_work); struct bio *bio;
while (1) { spin_lock(&bs->rescue_lock); bio = bio_list_pop(&bs->rescue_list); spin_unlock(&bs->rescue_lock);
if (!bio) break;
submit_bio_noacct(bio); }}
/* * submit_bio_noacct() converts recursion to iteration; this means if we're * running beneath it, any bios we allocate and submit will not be submitted * (and thus freed) until after we return. * * This exposes us to a potential deadlock if we allocate multiple bios from the * same bio_set while running underneath submit_bio_noacct(). If we were to * allocate multiple bios (say a stacking block driver that was splitting bios), * we would deadlock if we exhausted the mempool's reserve. * * We solve this, and guarantee forward progress by punting the bios on * current->bio_list to a per bio_set rescuer workqueue before blocking to wait * for elements being returned to the mempool. */static void punt_bios_to_rescuer(struct bio_set *bs){ struct bio_list punt, nopunt; struct bio *bio;
if (!current->bio_list || !bs->rescue_workqueue) return; if (bio_list_empty(¤t->bio_list[0]) && bio_list_empty(¤t->bio_list[1])) return;
/* * In order to guarantee forward progress we must punt only bios that * were allocated from this bio_set; otherwise, if there was a bio on * there for a stacking driver higher up in the stack, processing it * could require allocating bios from this bio_set, and doing that from * our own rescuer would be bad. * * Since bio lists are singly linked, pop them all instead of trying to * remove from the middle of the list: */
bio_list_init(&punt); bio_list_init(&nopunt);
while ((bio = bio_list_pop(¤t->bio_list[0]))) bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio); current->bio_list[0] = nopunt;
bio_list_init(&nopunt); while ((bio = bio_list_pop(¤t->bio_list[1]))) bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio); current->bio_list[1] = nopunt;
spin_lock(&bs->rescue_lock); bio_list_merge(&bs->rescue_list, &punt); spin_unlock(&bs->rescue_lock);
queue_work(bs->rescue_workqueue, &bs->rescue_work);}
static void bio_alloc_irq_cache_splice(struct bio_alloc_cache *cache){ unsigned long flags;
/* cache->free_list must be empty */ if (WARN_ON_ONCE(cache->free_list)) return;
local_irq_save(flags); cache->free_list = cache->free_list_irq; cache->free_list_irq = NULL; cache->nr += cache->nr_irq; cache->nr_irq = 0; local_irq_restore(flags);}
static struct bio *bio_alloc_percpu_cache(struct bio_set *bs){ struct bio_alloc_cache *cache; struct bio *bio;
cache = per_cpu_ptr(bs->cache, get_cpu()); if (!cache->free_list) { if (READ_ONCE(cache->nr_irq) >= ALLOC_CACHE_THRESHOLD) bio_alloc_irq_cache_splice(cache); if (!cache->free_list) { put_cpu(); return NULL; } } bio = cache->free_list; cache->free_list = bio->bi_next; cache->nr--; put_cpu(); bio->bi_pool = bs;
kmemleak_alloc(bio_slab_addr(bio), kmem_cache_size(bs->bio_slab), 1, GFP_NOIO); return bio;}
/** * bio_alloc_bioset - allocate a bio for I/O * @bdev: block device to allocate the bio for (can be %NULL) * @nr_vecs: number of bvecs to pre-allocate * @opf: operation and flags for bio * @gfp: the GFP_* mask given to the slab allocator * @bs: the bio_set to allocate from. * * Allocate a bio from the mempools in @bs. * * If %__GFP_DIRECT_RECLAIM is set then bio_alloc will always be able to * allocate a bio. This is due to the mempool guarantees. To make this work, * callers must never allocate more than 1 bio at a time from the general pool. * Callers that need to allocate more than 1 bio must always submit the * previously allocated bio for IO before attempting to allocate a new one. * Failure to do so can cause deadlocks under memory pressure. * * Note that when running under submit_bio_noacct() (i.e. any block driver), * bios are not submitted until after you return - see the code in * submit_bio_noacct() that converts recursion into iteration, to prevent * stack overflows. * * This would normally mean allocating multiple bios under submit_bio_noacct() * would be susceptible to deadlocks, but we have * deadlock avoidance code that resubmits any blocked bios from a rescuer * thread. * * However, we do not guarantee forward progress for allocations from other * mempools. Doing multiple allocations from the same mempool under * submit_bio_noacct() should be avoided - instead, use bio_set's front_pad * for per bio allocations. * * Returns: Pointer to new bio on success, NULL on failure. */struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs, blk_opf_t opf, gfp_t gfp, struct bio_set *bs){ struct bio_vec *bvecs = NULL; struct bio *bio = NULL; gfp_t saved_gfp = gfp; void *p;
/* should not use nobvec bioset for nr_vecs > 0 */ if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) && nr_vecs > 0)) return NULL;
if (saved_gfp & __GFP_DIRECT_RECLAIM) gfp = try_alloc_gfp(gfp); if (bs->cache && nr_vecs <= BIO_INLINE_VECS) { /* * Set REQ_ALLOC_CACHE even if no cached bio is available to * return the allocated bio to the percpu cache when done. */ opf |= REQ_ALLOC_CACHE; bio = bio_alloc_percpu_cache(bs); } else { opf &= ~REQ_ALLOC_CACHE; }
/* * For a bioset without a percpu cache, or when the percpu cache was * empty, try a slab allocation with optimistic GFP_ flags before * falling back to the mempool. */ if (!bio) { p = kmem_cache_alloc(bs->bio_slab, gfp); if (p) bio = p + bs->front_pad; }
if (bio && nr_vecs > BIO_INLINE_VECS) { struct biovec_slab *bvs = biovec_slab(nr_vecs);
/* * Upgrade nr_vecs to take full advantage of the allocation. * We also rely on this in bio_free(). */ nr_vecs = bvs->nr_vecs; bvecs = kmem_cache_alloc(bvs->slab, gfp); if (unlikely(!bvecs)) { kmem_cache_free(bs->bio_slab, p); bio = NULL; } }
if (unlikely(!bio)) { /* * Give up if we are not allow to sleep as non-blocking mempool * allocations just go back to the slab allocation. */ if (!(saved_gfp & __GFP_DIRECT_RECLAIM)) return NULL;
punt_bios_to_rescuer(bs);
/* * Don't rob the mempools by returning to the per-CPU cache if * we're tight on memory. */ opf &= ~REQ_ALLOC_CACHE;
p = mempool_alloc(&bs->bio_pool, saved_gfp); bio = p + bs->front_pad; if (nr_vecs > BIO_INLINE_VECS) { nr_vecs = BIO_MAX_VECS; bvecs = mempool_alloc(&bs->bvec_pool, saved_gfp); } }
if (nr_vecs && nr_vecs <= BIO_INLINE_VECS) bio_init_inline(bio, bdev, nr_vecs, opf); else bio_init(bio, bdev, bvecs, nr_vecs, opf); bio->bi_pool = bs; return bio;}EXPORT_SYMBOL(bio_alloc_bioset);
/** * bio_kmalloc - kmalloc a bio * @nr_vecs: number of bio_vecs to allocate * @gfp_mask: the GFP_* mask given to the slab allocator * * Use kmalloc to allocate a bio (including bvecs). The bio must be initialized * using bio_init() before use. To free a bio returned from this function use * kfree() after calling bio_uninit(). A bio returned from this function can * be reused by calling bio_uninit() before calling bio_init() again. * * Note that unlike bio_alloc() or bio_alloc_bioset() allocations from this * function are not backed by a mempool can fail. Do not use this function * for allocations in the file system I/O path. * * Returns: Pointer to new bio on success, NULL on failure. */struct bio *bio_kmalloc(unsigned short nr_vecs, gfp_t gfp_mask){ struct bio *bio;
if (nr_vecs > BIO_MAX_INLINE_VECS) return NULL; return kmalloc(sizeof(*bio) + nr_vecs * sizeof(struct bio_vec), gfp_mask);}EXPORT_SYMBOL(bio_kmalloc);
void zero_fill_bio(struct bio *bio){ struct bio_vec bv; struct bvec_iter iter;
bio_for_each_segment(bv, bio, iter) memzero_bvec(&bv);}EXPORT_SYMBOL(zero_fill_bio);
/** * bio_truncate - truncate the bio to small size of @new_size * @bio: the bio to be truncated * @new_size: new size for truncating the bio * * Description: * Truncate the bio to new size of @new_size. If bio_op(bio) is * REQ_OP_READ, zero the truncated part. This function should only * be used for handling corner cases, such as bio eod. */static void bio_truncate(struct bio *bio, unsigned new_size){ struct bio_vec bv; struct bvec_iter iter; unsigned int done = 0; bool truncated = false;
if (new_size >= bio->bi_iter.bi_size) return;
if (bio_op(bio) != REQ_OP_READ) goto exit;
bio_for_each_segment(bv, bio, iter) { if (done + bv.bv_len > new_size) { size_t offset;
if (!truncated) offset = new_size - done; else offset = 0; memzero_page(bv.bv_page, bv.bv_offset + offset, bv.bv_len - offset); truncated = true; } done += bv.bv_len; }
exit: /* * Don't touch bvec table here and make it really immutable, since * fs bio user has to retrieve all pages via bio_for_each_segment_all * in its .end_bio() callback. * * It is enough to truncate bio by updating .bi_size since we can make * correct bvec with the updated .bi_size for drivers. */ bio->bi_iter.bi_size = new_size;}
/** * guard_bio_eod - truncate a BIO to fit the block device * @bio: bio to truncate * * This allows us to do IO even on the odd last sectors of a device, even if the * block size is some multiple of the physical sector size. * * We'll just truncate the bio to the size of the device, and clear the end of * the buffer head manually. Truly out-of-range accesses will turn into actual * I/O errors, this only handles the "we need to be able to do I/O at the final * sector" case. */void guard_bio_eod(struct bio *bio){ sector_t maxsector = bdev_nr_sectors(bio->bi_bdev);
if (!maxsector) return;
/* * If the *whole* IO is past the end of the device, * let it through, and the IO layer will turn it into * an EIO. */ if (unlikely(bio->bi_iter.bi_sector >= maxsector)) return;
maxsector -= bio->bi_iter.bi_sector; if (likely((bio->bi_iter.bi_size >> 9) <= maxsector)) return;
bio_truncate(bio, maxsector << 9);}
static int __bio_alloc_cache_prune(struct bio_alloc_cache *cache, unsigned int nr){ unsigned int i = 0; struct bio *bio;
while ((bio = cache->free_list) != NULL) { cache->free_list = bio->bi_next; cache->nr--; kmemleak_alloc(bio_slab_addr(bio), kmem_cache_size(bio->bi_pool->bio_slab), 1, GFP_KERNEL); bio_free(bio); if (++i == nr) break; } return i;}
static void bio_alloc_cache_prune(struct bio_alloc_cache *cache, unsigned int nr){ nr -= __bio_alloc_cache_prune(cache, nr); if (!READ_ONCE(cache->free_list)) { bio_alloc_irq_cache_splice(cache); __bio_alloc_cache_prune(cache, nr); }}
static int bio_cpu_dead(unsigned int cpu, struct hlist_node *node){ struct bio_set *bs;
bs = hlist_entry_safe(node, struct bio_set, cpuhp_dead); if (bs->cache) { struct bio_alloc_cache *cache = per_cpu_ptr(bs->cache, cpu);
bio_alloc_cache_prune(cache, -1U); } return 0;}
static void bio_alloc_cache_destroy(struct bio_set *bs){ int cpu;
if (!bs->cache) return;
cpuhp_state_remove_instance_nocalls(CPUHP_BIO_DEAD, &bs->cpuhp_dead); for_each_possible_cpu(cpu) { struct bio_alloc_cache *cache;
cache = per_cpu_ptr(bs->cache, cpu); bio_alloc_cache_prune(cache, -1U); } free_percpu(bs->cache); bs->cache = NULL;}
static inline void bio_put_percpu_cache(struct bio *bio){ struct bio_alloc_cache *cache;
cache = per_cpu_ptr(bio->bi_pool->cache, get_cpu()); if (READ_ONCE(cache->nr_irq) + cache->nr > ALLOC_CACHE_MAX) goto out_free;
if (in_task()) { bio_uninit(bio); bio->bi_next = cache->free_list; /* Not necessary but helps not to iopoll already freed bios */ bio->bi_bdev = NULL; cache->free_list = bio; cache->nr++; kmemleak_free(bio_slab_addr(bio)); } else if (in_hardirq()) { lockdep_assert_irqs_disabled();
bio_uninit(bio); bio->bi_next = cache->free_list_irq; cache->free_list_irq = bio; cache->nr_irq++; kmemleak_free(bio_slab_addr(bio)); } else { goto out_free; } put_cpu(); return;out_free: put_cpu(); bio_free(bio);}
/** * bio_put - release a reference to a bio * @bio: bio to release reference to * * Description: * Put a reference to a &struct bio, either one you have gotten with * bio_alloc, bio_get or bio_clone_*. The last put of a bio will free it. **/void bio_put(struct bio *bio){ if (unlikely(bio_flagged(bio, BIO_REFFED))) { BUG_ON(!atomic_read(&bio->__bi_cnt)); if (!atomic_dec_and_test(&bio->__bi_cnt)) return; } if (bio->bi_opf & REQ_ALLOC_CACHE) bio_put_percpu_cache(bio); else bio_free(bio);}EXPORT_SYMBOL(bio_put);
static int __bio_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp){ bio_set_flag(bio, BIO_CLONED); bio->bi_ioprio = bio_src->bi_ioprio; bio->bi_write_hint = bio_src->bi_write_hint; bio->bi_write_stream = bio_src->bi_write_stream; bio->bi_iter = bio_src->bi_iter;
if (bio->bi_bdev) { if (bio->bi_bdev == bio_src->bi_bdev && bio_flagged(bio_src, BIO_REMAPPED)) bio_set_flag(bio, BIO_REMAPPED); bio_clone_blkg_association(bio, bio_src); }
if (bio_crypt_clone(bio, bio_src, gfp) < 0) return -ENOMEM; if (bio_integrity(bio_src) && bio_integrity_clone(bio, bio_src, gfp) < 0) return -ENOMEM; return 0;}
/** * bio_alloc_clone - clone a bio that shares the original bio's biovec * @bdev: block_device to clone onto * @bio_src: bio to clone from * @gfp: allocation priority * @bs: bio_set to allocate from * * Allocate a new bio that is a clone of @bio_src. This reuses the bio_vecs * pointed to by @bio_src->bi_io_vec, and clones the iterator pointing to * the current position in it. The caller owns the returned bio, but not * the bio_vecs, and must ensure the bio is freed before the memory * pointed to by @bio_Src->bi_io_vecs. */struct bio *bio_alloc_clone(struct block_device *bdev, struct bio *bio_src, gfp_t gfp, struct bio_set *bs){ struct bio *bio;
bio = bio_alloc_bioset(bdev, 0, bio_src->bi_opf, gfp, bs); if (!bio) return NULL;
if (__bio_clone(bio, bio_src, gfp) < 0) { bio_put(bio); return NULL; } bio->bi_io_vec = bio_src->bi_io_vec;
return bio;}EXPORT_SYMBOL(bio_alloc_clone);
/** * bio_init_clone - clone a bio that shares the original bio's biovec * @bdev: block_device to clone onto * @bio: bio to clone into * @bio_src: bio to clone from * @gfp: allocation priority * * Initialize a new bio in caller provided memory that is a clone of @bio_src. * The same bio_vecs reuse and bio lifetime rules as bio_alloc_clone() apply. */int bio_init_clone(struct block_device *bdev, struct bio *bio, struct bio *bio_src, gfp_t gfp){ int ret;
bio_init(bio, bdev, bio_src->bi_io_vec, 0, bio_src->bi_opf); ret = __bio_clone(bio, bio_src, gfp); if (ret) bio_uninit(bio); return ret;}EXPORT_SYMBOL(bio_init_clone);
/** * bio_full - check if the bio is full * @bio: bio to check * @len: length of one segment to be added * * Return true if @bio is full and one segment with @len bytes can't be * added to the bio, otherwise return false */static inline bool bio_full(struct bio *bio, unsigned len){ if (bio->bi_vcnt >= bio->bi_max_vecs) return true; if (bio->bi_iter.bi_size > BIO_MAX_SIZE - len) return true; return false;}
static bool bvec_try_merge_page(struct bio_vec *bv, struct page *page, unsigned int len, unsigned int off){ size_t bv_end = bv->bv_offset + bv->bv_len; phys_addr_t vec_end_addr = page_to_phys(bv->bv_page) + bv_end - 1; phys_addr_t page_addr = page_to_phys(page);
if (vec_end_addr + 1 != page_addr + off) return false; if (xen_domain() && !xen_biovec_phys_mergeable(bv, page)) return false;
if ((vec_end_addr & PAGE_MASK) != ((page_addr + off) & PAGE_MASK)) { if (IS_ENABLED(CONFIG_KMSAN)) return false; if (bv->bv_page + bv_end / PAGE_SIZE != page + off / PAGE_SIZE) return false; }
bv->bv_len += len; return true;}
/* * Try to merge a page into a segment, while obeying the hardware segment * size limit. * * This is kept around for the integrity metadata, which is still tries * to build the initial bio to the hardware limit and doesn't have proper * helpers to split. Hopefully this will go away soon. */bool bvec_try_merge_hw_page(struct request_queue *q, struct bio_vec *bv, struct page *page, unsigned len, unsigned offset){ unsigned long mask = queue_segment_boundary(q); phys_addr_t addr1 = bvec_phys(bv); phys_addr_t addr2 = page_to_phys(page) + offset + len - 1;
if ((addr1 | mask) != (addr2 | mask)) return false; if (len > queue_max_segment_size(q) - bv->bv_len) return false; return bvec_try_merge_page(bv, page, len, offset);}
/** * __bio_add_page - add page(s) to a bio in a new segment * @bio: destination bio * @page: start page to add * @len: length of the data to add, may cross pages * @off: offset of the data relative to @page, may cross pages * * Add the data at @page + @off to @bio as a new bvec. The caller must ensure * that @bio has space for another bvec. */void __bio_add_page(struct bio *bio, struct page *page, unsigned int len, unsigned int off){ WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)); WARN_ON_ONCE(bio_full(bio, len));
if (is_pci_p2pdma_page(page)) bio->bi_opf |= REQ_NOMERGE;
bvec_set_page(&bio->bi_io_vec[bio->bi_vcnt], page, len, off); bio->bi_iter.bi_size += len; bio->bi_vcnt++;}EXPORT_SYMBOL_GPL(__bio_add_page);
/** * bio_add_virt_nofail - add data in the direct kernel mapping to a bio * @bio: destination bio * @vaddr: data to add * @len: length of the data to add, may cross pages * * Add the data at @vaddr to @bio. The caller must have ensure a segment * is available for the added data. No merging into an existing segment * will be performed. */void bio_add_virt_nofail(struct bio *bio, void *vaddr, unsigned len){ __bio_add_page(bio, virt_to_page(vaddr), len, offset_in_page(vaddr));}EXPORT_SYMBOL_GPL(bio_add_virt_nofail);
/** * bio_add_page - attempt to add page(s) to bio * @bio: destination bio * @page: start page to add * @len: vec entry length, may cross pages * @offset: vec entry offset relative to @page, may cross pages * * Attempt to add page(s) to the bio_vec maplist. This will only fail * if either bio->bi_vcnt == bio->bi_max_vecs or it's a cloned bio. */int bio_add_page(struct bio *bio, struct page *page, unsigned int len, unsigned int offset){ if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) return 0; if (WARN_ON_ONCE(len == 0)) return 0; if (bio->bi_iter.bi_size > BIO_MAX_SIZE - len) return 0;
if (bio->bi_vcnt > 0) { struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
if (!zone_device_pages_compatible(bv->bv_page, page)) return 0; if (zone_device_pages_have_same_pgmap(bv->bv_page, page) && bvec_try_merge_page(bv, page, len, offset)) { bio->bi_iter.bi_size += len; return len; } }
if (bio->bi_vcnt >= bio->bi_max_vecs) return 0; __bio_add_page(bio, page, len, offset); return len;}EXPORT_SYMBOL(bio_add_page);
void bio_add_folio_nofail(struct bio *bio, struct folio *folio, size_t len, size_t off){ unsigned long nr = off / PAGE_SIZE;
WARN_ON_ONCE(len > BIO_MAX_SIZE); __bio_add_page(bio, folio_page(folio, nr), len, off % PAGE_SIZE);}EXPORT_SYMBOL_GPL(bio_add_folio_nofail);
/** * bio_add_folio - Attempt to add part of a folio to a bio. * @bio: BIO to add to. * @folio: Folio to add. * @len: How many bytes from the folio to add. * @off: First byte in this folio to add. * * Filesystems that use folios can call this function instead of calling * bio_add_page() for each page in the folio. If @off is bigger than * PAGE_SIZE, this function can create a bio_vec that starts in a page * after the bv_page. BIOs do not support folios that are 4GiB or larger. * * Return: Whether the addition was successful. */bool bio_add_folio(struct bio *bio, struct folio *folio, size_t len, size_t off){ unsigned long nr = off / PAGE_SIZE;
if (len > BIO_MAX_SIZE) return false; return bio_add_page(bio, folio_page(folio, nr), len, off % PAGE_SIZE) > 0;}EXPORT_SYMBOL(bio_add_folio);
/** * bio_add_vmalloc_chunk - add a vmalloc chunk to a bio * @bio: destination bio * @vaddr: vmalloc address to add * @len: total length in bytes of the data to add * * Add data starting at @vaddr to @bio and return how many bytes were added. * This may be less than the amount originally asked. Returns 0 if no data * could be added to @bio. * * This helper calls flush_kernel_vmap_range() for the range added. For reads * the caller still needs to manually call invalidate_kernel_vmap_range() in * the completion handler. */unsigned int bio_add_vmalloc_chunk(struct bio *bio, void *vaddr, unsigned len){ unsigned int offset = offset_in_page(vaddr);
len = min(len, PAGE_SIZE - offset); if (bio_add_page(bio, vmalloc_to_page(vaddr), len, offset) < len) return 0; if (op_is_write(bio_op(bio))) flush_kernel_vmap_range(vaddr, len); return len;}EXPORT_SYMBOL_GPL(bio_add_vmalloc_chunk);
/** * bio_add_vmalloc - add a vmalloc region to a bio * @bio: destination bio * @vaddr: vmalloc address to add * @len: total length in bytes of the data to add * * Add data starting at @vaddr to @bio. Return %true on success or %false if * @bio does not have enough space for the payload. * * This helper calls flush_kernel_vmap_range() for the range added. For reads * the caller still needs to manually call invalidate_kernel_vmap_range() in * the completion handler. */bool bio_add_vmalloc(struct bio *bio, void *vaddr, unsigned int len){ do { unsigned int added = bio_add_vmalloc_chunk(bio, vaddr, len);
if (!added) return false; vaddr += added; len -= added; } while (len);
return true;}EXPORT_SYMBOL_GPL(bio_add_vmalloc);
void __bio_release_pages(struct bio *bio, bool mark_dirty){ struct folio_iter fi;
bio_for_each_folio_all(fi, bio) { size_t nr_pages;
if (mark_dirty) { folio_lock(fi.folio); folio_mark_dirty(fi.folio); folio_unlock(fi.folio); } nr_pages = (fi.offset + fi.length - 1) / PAGE_SIZE - fi.offset / PAGE_SIZE + 1; unpin_user_folio(fi.folio, nr_pages); }}EXPORT_SYMBOL_GPL(__bio_release_pages);
void bio_iov_bvec_set(struct bio *bio, const struct iov_iter *iter){ WARN_ON_ONCE(bio->bi_max_vecs);
bio->bi_io_vec = (struct bio_vec *)iter->bvec; bio->bi_iter.bi_idx = 0; bio->bi_iter.bi_bvec_done = iter->iov_offset; bio->bi_iter.bi_size = iov_iter_count(iter); bio_set_flag(bio, BIO_CLONED);}
/* * Aligns the bio size to the len_align_mask, releasing excessive bio vecs that * __bio_iov_iter_get_pages may have inserted, and reverts the trimmed length * for the next iteration. */static int bio_iov_iter_align_down(struct bio *bio, struct iov_iter *iter, struct bio_vec *bv, unsigned len_align_mask){ size_t nbytes = bio->bi_iter.bi_size & len_align_mask;
if (!nbytes) return 0;
iov_iter_revert(iter, nbytes); bio->bi_iter.bi_size -= nbytes; while (nbytes >= bv->bv_len) { if (bio_flagged(bio, BIO_PAGE_PINNED)) unpin_user_page(bv->bv_page);
if (!--bio->bi_vcnt) return -EFAULT; nbytes -= bv->bv_len; bv--; } bv->bv_len -= nbytes; return 0;}
/** * bio_iov_iter_get_pages - add user or kernel pages to a bio * @bio: bio to add pages to * @iter: iov iterator describing the region to be added * @len_align_mask: the mask to align the total size to, 0 for any length * * This takes either an iterator pointing to user memory, or one pointing to * kernel pages (BVEC iterator). If we're adding user pages, we pin them and * map them into the kernel. On IO completion, the caller should put those * pages. For bvec based iterators bio_iov_iter_get_pages() uses the provided * bvecs rather than copying them. Hence anyone issuing kiocb based IO needs * to ensure the bvecs and pages stay referenced until the submitted I/O is * completed by a call to ->ki_complete() or returns with an error other than * -EIOCBQUEUED. The caller needs to check if the bio is flagged BIO_NO_PAGE_REF * on IO completion. If it isn't, then pages should be released. * * The function tries, but does not guarantee, to pin as many pages as * fit into the bio, or are requested in @iter, whatever is smaller. If * MM encounters an error pinning the requested pages, it stops. Error * is returned only if 0 pages could be pinned. */int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, unsigned len_align_mask){ iov_iter_extraction_t flags = 0;
if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) return -EIO;
if (iov_iter_is_bvec(iter)) { bio_iov_bvec_set(bio, iter); iov_iter_advance(iter, bio->bi_iter.bi_size); return 0; }
if (iov_iter_extract_will_pin(iter)) bio_set_flag(bio, BIO_PAGE_PINNED); if (bio->bi_bdev && blk_queue_pci_p2pdma(bio->bi_bdev->bd_disk->queue)) flags |= ITER_ALLOW_P2PDMA;
do { ssize_t ret;
ret = iov_iter_extract_bvecs(iter, bio->bi_io_vec, BIO_MAX_SIZE - bio->bi_iter.bi_size, &bio->bi_vcnt, bio->bi_max_vecs, flags); if (ret <= 0) { if (!bio->bi_vcnt) return ret; break; } bio->bi_iter.bi_size += ret; } while (iov_iter_count(iter) && !bio_full(bio, 0));
if (is_pci_p2pdma_page(bio->bi_io_vec->bv_page)) bio->bi_opf |= REQ_NOMERGE; return bio_iov_iter_align_down(bio, iter, &bio->bi_io_vec[bio->bi_vcnt - 1], len_align_mask);}
static struct folio *folio_alloc_greedy(gfp_t gfp, size_t *size, size_t minsize){ struct folio *folio;
while (*size > minsize) { folio = folio_alloc(gfp | __GFP_NORETRY | __GFP_NOWARN, get_order(*size)); if (folio) return folio; *size = rounddown_pow_of_two(*size - 1); }
return folio_alloc(gfp, get_order(*size));}
static void bio_free_folios(struct bio *bio){ struct bio_vec *bv; int i;
bio_for_each_bvec_all(bv, bio, i) { struct folio *folio = bvec_folio(bv);
if (!is_zero_folio(folio) && !is_huge_zero_folio(folio)) folio_put(folio); }}
static int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter, size_t maxlen, size_t minsize){ size_t total_len = min(maxlen, iov_iter_count(iter));
if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) return -EINVAL; if (WARN_ON_ONCE(bio->bi_iter.bi_size)) return -EINVAL; if (WARN_ON_ONCE(bio->bi_vcnt >= bio->bi_max_vecs)) return -EINVAL;
do { size_t this_len = min(total_len, SZ_1M); size_t copied; struct folio *folio;
if (this_len > minsize * 2) this_len = rounddown_pow_of_two(this_len);
if (bio->bi_iter.bi_size > BIO_MAX_SIZE - this_len) break;
folio = folio_alloc_greedy(GFP_KERNEL, &this_len, minsize); if (!folio) break; bio_add_folio_nofail(bio, folio, this_len, 0);
if (iter->nofault) copied = copy_folio_from_iter_atomic(folio, 0, this_len, iter); else copied = copy_folio_from_iter(folio, 0, this_len, iter); if (copied < this_len) { /* * Need to revert the iov iter for all bytes we have * copied. * * However the bio size differs from the real copied * bytes as @this_len is queued but only advanced * less than that. * Need to compensate that for the revert. */ iov_iter_revert(iter, bio->bi_iter.bi_size - this_len + copied); bio_free_folios(bio); return -EFAULT; } total_len -= this_len; } while (total_len && bio->bi_vcnt < bio->bi_max_vecs);
if (!bio->bi_iter.bi_size) return -ENOMEM; return bio_iov_iter_align_down(bio, iter, &bio->bi_io_vec[bio->bi_vcnt - 1], minsize - 1);}
static int bio_iov_iter_bounce_read(struct bio *bio, struct iov_iter *iter, size_t maxlen, size_t minsize){ size_t len = min3(iov_iter_count(iter), maxlen, SZ_1M); struct folio *folio; ssize_t ret;
folio = folio_alloc_greedy(GFP_KERNEL, &len, minsize); if (!folio) return -ENOMEM;
do { ret = iov_iter_extract_bvecs(iter, bio->bi_io_vec + 1, len, &bio->bi_vcnt, bio->bi_max_vecs - 1, 0); if (ret <= 0) { if (!bio->bi_vcnt) goto out_folio_put; break; } len -= ret; bio->bi_iter.bi_size += ret; } while (len && bio->bi_vcnt < bio->bi_max_vecs - 1);
/* * Set the folio directly here. The above loop has already calculated * the correct bi_size, and we use bi_vcnt for the user buffers. That * is safe as bi_vcnt is only used by the submitter and not the actual * I/O path. */ bvec_set_folio(&bio->bi_io_vec[0], folio, bio->bi_iter.bi_size, 0); if (iov_iter_extract_will_pin(iter)) bio_set_flag(bio, BIO_PAGE_PINNED);
/* The first vec stores the bounce buffer, so do not subtract 1 here. */ ret = bio_iov_iter_align_down(bio, iter, &bio->bi_io_vec[bio->bi_vcnt], minsize - 1); if (ret) goto out_folio_put;
/* Update the bounc buffer bv_len to the aligned down size. */ bio->bi_io_vec[0].bv_len = bio->bi_iter.bi_size; return 0;
out_folio_put: folio_put(folio); return ret;}
/** * bio_iov_iter_bounce - bounce buffer data from an iter into a bio * @bio: bio to send * @iter: iter to read from / write into * @maxlen: maximum size to bounce * @minsize: minimum folio allocation size * * Helper for direct I/O implementations that need to bounce buffer because * we need to checksum the data or perform other operations that require * consistency. Allocates folios to back the bounce buffer, and for writes * copies the data into it. Needs to be paired with bio_iov_iter_unbounce() * called on completion. */int bio_iov_iter_bounce(struct bio *bio, struct iov_iter *iter, size_t maxlen, size_t minsize){ if (op_is_write(bio_op(bio))) return bio_iov_iter_bounce_write(bio, iter, maxlen, minsize); return bio_iov_iter_bounce_read(bio, iter, maxlen, minsize);}
static void bvec_unpin(struct bio_vec *bv, bool mark_dirty){ struct folio *folio = bvec_folio(bv); size_t nr_pages = (bv->bv_offset + bv->bv_len - 1) / PAGE_SIZE - bv->bv_offset / PAGE_SIZE + 1;
if (mark_dirty) folio_mark_dirty_lock(folio); unpin_user_folio(folio, nr_pages);}
static void bio_iov_iter_unbounce_read(struct bio *bio, bool is_error, bool mark_dirty){ unsigned int len = bio->bi_io_vec[0].bv_len;
if (likely(!is_error)) { void *buf = bvec_virt(&bio->bi_io_vec[0]); struct iov_iter to;
iov_iter_bvec(&to, ITER_DEST, bio->bi_io_vec + 1, bio->bi_vcnt, len); /* copying to pinned pages should always work */ WARN_ON_ONCE(copy_to_iter(buf, len, &to) != len); } else { /* No need to mark folios dirty if never copied to them */ mark_dirty = false; }
if (bio_flagged(bio, BIO_PAGE_PINNED)) { int i;
for (i = 0; i < bio->bi_vcnt; i++) bvec_unpin(&bio->bi_io_vec[1 + i], mark_dirty); }
folio_put(bvec_folio(&bio->bi_io_vec[0]));}
/** * bio_iov_iter_unbounce - finish a bounce buffer operation * @bio: completed bio * @is_error: %true if an I/O error occurred and data should not be copied * @mark_dirty: If %true, folios will be marked dirty. * * Helper for direct I/O implementations that need to bounce buffer because * we need to checksum the data or perform other operations that require * consistency. Called to complete a bio set up by bio_iov_iter_bounce(). * Copies data back for reads, and marks the original folios dirty if * requested and then frees the bounce buffer. */void bio_iov_iter_unbounce(struct bio *bio, bool is_error, bool mark_dirty){ if (op_is_write(bio_op(bio))) bio_free_folios(bio); else bio_iov_iter_unbounce_read(bio, is_error, mark_dirty);}
static void bio_wait_end_io(struct bio *bio){ complete(bio->bi_private);}
/** * bio_await - call a function on a bio, and wait until it completes * @bio: the bio which describes the I/O * @submit: function called to submit the bio * @priv: private data passed to @submit * * Wait for the bio as well as any bio chained off it after executing the * passed in callback @submit. The wait for the bio is set up before calling * @submit to ensure that the completion is captured. If @submit is %NULL, * submit_bio() is used instead to submit the bio. * * Note: this overrides the bi_private and bi_end_io fields in the bio. */void bio_await(struct bio *bio, void *priv, void (*submit)(struct bio *bio, void *priv)){ DECLARE_COMPLETION_ONSTACK_MAP(done, bio->bi_bdev->bd_disk->lockdep_map);
bio->bi_private = &done; bio->bi_end_io = bio_wait_end_io; bio->bi_opf |= REQ_SYNC; if (submit) submit(bio, priv); else submit_bio(bio); blk_wait_io(&done);}EXPORT_SYMBOL_GPL(bio_await);
/** * submit_bio_wait - submit a bio, and wait until it completes * @bio: The &struct bio which describes the I/O * * Simple wrapper around submit_bio(). Returns 0 on success, or the error from * bio_endio() on failure. * * WARNING: Unlike to how submit_bio() is usually used, this function does not * result in bio reference to be consumed. The caller must drop the reference * on his own. */int submit_bio_wait(struct bio *bio){ bio_await(bio, NULL, NULL); return blk_status_to_errno(bio->bi_status);}EXPORT_SYMBOL(submit_bio_wait);
static void bio_endio_cb(struct bio *bio, void *priv){ bio_endio(bio);}
/* * Submit @bio synchronously, or call bio_endio on it if the current process * is being killed. */int bio_submit_or_kill(struct bio *bio, unsigned int flags){ if ((flags & BLKDEV_ZERO_KILLABLE) && fatal_signal_pending(current)) { bio_await(bio, NULL, bio_endio_cb); return -EINTR; }
return submit_bio_wait(bio);}
/** * bdev_rw_virt - synchronously read into / write from kernel mapping * @bdev: block device to access * @sector: sector to access * @data: data to read/write * @len: length in byte to read/write * @op: operation (e.g. REQ_OP_READ/REQ_OP_WRITE) * * Performs synchronous I/O to @bdev for @data/@len. @data must be in * the kernel direct mapping and not a vmalloc address. */int bdev_rw_virt(struct block_device *bdev, sector_t sector, void *data, size_t len, enum req_op op){ struct bio_vec bv; struct bio bio; int error;
if (WARN_ON_ONCE(is_vmalloc_addr(data))) return -EIO;
bio_init(&bio, bdev, &bv, 1, op); bio.bi_iter.bi_sector = sector; bio_add_virt_nofail(&bio, data, len); error = submit_bio_wait(&bio); bio_uninit(&bio); return error;}EXPORT_SYMBOL_GPL(bdev_rw_virt);
void __bio_advance(struct bio *bio, unsigned bytes){ if (bio_integrity(bio)) bio_integrity_advance(bio, bytes);
bio_crypt_advance(bio, bytes); bio_advance_iter(bio, &bio->bi_iter, bytes);}EXPORT_SYMBOL(__bio_advance);
/** * bio_copy_data - copy contents of data buffers from one bio to another * @src: source bio * @dst: destination bio * * Stops when it reaches the end of either @src or @dst - that is, copies * min(src->bi_size, dst->bi_size) bytes (or the equivalent for lists of bios). */void bio_copy_data(struct bio *dst, struct bio *src){ struct bvec_iter src_iter = src->bi_iter; struct bvec_iter dst_iter = dst->bi_iter;
while (src_iter.bi_size && dst_iter.bi_size) { struct bio_vec src_bv = bio_iter_iovec(src, src_iter); struct bio_vec dst_bv = bio_iter_iovec(dst, dst_iter); unsigned int bytes = min(src_bv.bv_len, dst_bv.bv_len); void *src_buf = bvec_kmap_local(&src_bv); void *dst_buf = bvec_kmap_local(&dst_bv);
memcpy(dst_buf, src_buf, bytes);
kunmap_local(dst_buf); kunmap_local(src_buf);
bio_advance_iter_single(src, &src_iter, bytes); bio_advance_iter_single(dst, &dst_iter, bytes); }}EXPORT_SYMBOL(bio_copy_data);
void bio_free_pages(struct bio *bio){ struct bio_vec *bvec; struct bvec_iter_all iter_all;
bio_for_each_segment_all(bvec, bio, iter_all) __free_page(bvec->bv_page);}EXPORT_SYMBOL(bio_free_pages);
/* * bio_set_pages_dirty() and bio_check_pages_dirty() are support functions * for performing direct-IO in BIOs. * * The problem is that we cannot run folio_mark_dirty() from interrupt context * because the required locks are not interrupt-safe. So what we can do is to * mark the pages dirty _before_ performing IO. And in interrupt context, * check that the pages are still dirty. If so, fine. If not, redirty them * in process context. * * Note that this code is very hard to test under normal circumstances because * direct-io pins the pages with get_user_pages(). This makes * is_page_cache_freeable return false, and the VM will not clean the pages. * But other code (eg, flusher threads) could clean the pages if they are mapped * pagecache. * * Simply disabling the call to bio_set_pages_dirty() is a good way to test the * deferred bio dirtying paths. */
/* * bio_set_pages_dirty() will mark all the bio's pages as dirty. */void bio_set_pages_dirty(struct bio *bio){ struct folio_iter fi;
bio_for_each_folio_all(fi, bio) { folio_lock(fi.folio); folio_mark_dirty(fi.folio); folio_unlock(fi.folio); }}
/* * bio_check_pages_dirty() will check that all the BIO's pages are still dirty. * If they are, then fine. If, however, some pages are clean then they must * have been written out during the direct-IO read. So we take another ref on * the BIO and re-dirty the pages in process context. * * It is expected that bio_check_pages_dirty() will wholly own the BIO from * here on. It will unpin each page and will run one bio_put() against the * BIO. */
static void bio_dirty_fn(struct work_struct *work);
static DECLARE_WORK(bio_dirty_work, bio_dirty_fn);static DEFINE_SPINLOCK(bio_dirty_lock);static struct bio *bio_dirty_list;
/* * This runs in process context */static void bio_dirty_fn(struct work_struct *work){ struct bio *bio, *next;
spin_lock_irq(&bio_dirty_lock); next = bio_dirty_list; bio_dirty_list = NULL; spin_unlock_irq(&bio_dirty_lock);
while ((bio = next) != NULL) { next = bio->bi_private;
bio_release_pages(bio, true); bio_put(bio); }}
void bio_check_pages_dirty(struct bio *bio){ struct folio_iter fi; unsigned long flags;
bio_for_each_folio_all(fi, bio) { if (!folio_test_dirty(fi.folio)) goto defer; }
bio_release_pages(bio, false); bio_put(bio); return;defer: spin_lock_irqsave(&bio_dirty_lock, flags); bio->bi_private = bio_dirty_list; bio_dirty_list = bio; spin_unlock_irqrestore(&bio_dirty_lock, flags); schedule_work(&bio_dirty_work);}
static inline bool bio_remaining_done(struct bio *bio){ /* * If we're not chaining, then ->__bi_remaining is always 1 and * we always end io on the first invocation. */ if (!bio_flagged(bio, BIO_CHAIN)) return true;
BUG_ON(atomic_read(&bio->__bi_remaining) <= 0);
if (atomic_dec_and_test(&bio->__bi_remaining)) { bio_clear_flag(bio, BIO_CHAIN); return true; }
return false;}
/** * bio_endio - end I/O on a bio * @bio: bio * * Description: * bio_endio() will end I/O on the whole bio. bio_endio() is the preferred * way to end I/O on a bio. No one should call bi_end_io() directly on a * bio unless they own it and thus know that it has an end_io function. * * bio_endio() can be called several times on a bio that has been chained * using bio_chain(). The ->bi_end_io() function will only be called the * last time. **/void bio_endio(struct bio *bio){again: if (!bio_remaining_done(bio)) return; if (!bio_integrity_endio(bio)) return;
blk_zone_bio_endio(bio);
rq_qos_done_bio(bio);
if (bio->bi_bdev && bio_flagged(bio, BIO_TRACE_COMPLETION)) { trace_block_bio_complete(bdev_get_queue(bio->bi_bdev), bio); bio_clear_flag(bio, BIO_TRACE_COMPLETION); }
/* * Need to have a real endio function for chained bios, otherwise * various corner cases will break (like stacking block devices that * save/restore bi_end_io) - however, we want to avoid unbounded * recursion and blowing the stack. Tail call optimization would * handle this, but compiling with frame pointers also disables * gcc's sibling call optimization. */ if (bio->bi_end_io == bio_chain_endio) { bio = __bio_chain_endio(bio); goto again; }
#ifdef CONFIG_BLK_CGROUP /* * Release cgroup info. We shouldn't have to do this here, but quite * a few callers of bio_init fail to call bio_uninit, so we cover up * for that here at least for now. */ if (bio->bi_blkg) { blkg_put(bio->bi_blkg); bio->bi_blkg = NULL; }#endif
if (bio->bi_end_io) bio->bi_end_io(bio);}EXPORT_SYMBOL(bio_endio);
/** * bio_split - split a bio * @bio: bio to split * @sectors: number of sectors to split from the front of @bio * @gfp: gfp mask * @bs: bio set to allocate from * * Allocates and returns a new bio which represents @sectors from the start of * @bio, and updates @bio to represent the remaining sectors. * * Unless this is a discard request the newly allocated bio will point * to @bio's bi_io_vec. It is the caller's responsibility to ensure that * neither @bio nor @bs are freed before the split bio. */struct bio *bio_split(struct bio *bio, int sectors, gfp_t gfp, struct bio_set *bs){ struct bio *split;
if (WARN_ON_ONCE(sectors <= 0)) return ERR_PTR(-EINVAL); if (WARN_ON_ONCE(sectors >= bio_sectors(bio))) return ERR_PTR(-EINVAL);
/* Zone append commands cannot be split */ if (WARN_ON_ONCE(bio_op(bio) == REQ_OP_ZONE_APPEND)) return ERR_PTR(-EINVAL);
/* atomic writes cannot be split */ if (bio->bi_opf & REQ_ATOMIC) return ERR_PTR(-EINVAL);
split = bio_alloc_clone(bio->bi_bdev, bio, gfp, bs); if (!split) return ERR_PTR(-ENOMEM);
split->bi_iter.bi_size = sectors << 9;
if (bio_integrity(split)) bio_integrity_trim(split);
bio_advance(bio, split->bi_iter.bi_size);
if (bio_flagged(bio, BIO_TRACE_COMPLETION)) bio_set_flag(split, BIO_TRACE_COMPLETION);
return split;}EXPORT_SYMBOL(bio_split);
/** * bio_trim - trim a bio * @bio: bio to trim * @offset: number of sectors to trim from the front of @bio * @size: size we want to trim @bio to, in sectors * * This function is typically used for bios that are cloned and submitted * to the underlying device in parts. */void bio_trim(struct bio *bio, sector_t offset, sector_t size){ /* We should never trim an atomic write */ if (WARN_ON_ONCE(bio->bi_opf & REQ_ATOMIC && size)) return;
if (WARN_ON_ONCE(offset > BIO_MAX_SECTORS || size > BIO_MAX_SECTORS || offset + size > bio_sectors(bio))) return;
size <<= 9; if (offset == 0 && size == bio->bi_iter.bi_size) return;
bio_advance(bio, offset << 9); bio->bi_iter.bi_size = size;
if (bio_integrity(bio)) bio_integrity_trim(bio);}EXPORT_SYMBOL_GPL(bio_trim);
/* * create memory pools for biovec's in a bio_set. * use the global biovec slabs created for general use. */static int biovec_init_pool(mempool_t *pool, int pool_entries){ struct biovec_slab *bp = bvec_slabs + ARRAY_SIZE(bvec_slabs) - 1;
return mempool_init_slab_pool(pool, pool_entries, bp->slab);}
/* * bioset_exit - exit a bioset initialized with bioset_init() * * May be called on a zeroed but uninitialized bioset (i.e. allocated with * kzalloc()). */void bioset_exit(struct bio_set *bs){ bio_alloc_cache_destroy(bs); if (bs->rescue_workqueue) destroy_workqueue(bs->rescue_workqueue); bs->rescue_workqueue = NULL;
mempool_exit(&bs->bio_pool); mempool_exit(&bs->bvec_pool);
if (bs->bio_slab) bio_put_slab(bs); bs->bio_slab = NULL;}EXPORT_SYMBOL(bioset_exit);
/** * bioset_init - Initialize a bio_set * @bs: pool to initialize * @pool_size: Number of bio and bio_vecs to cache in the mempool * @front_pad: Number of bytes to allocate in front of the returned bio * @flags: Flags to modify behavior, currently %BIOSET_NEED_BVECS * and %BIOSET_NEED_RESCUER * * Description: * Set up a bio_set to be used with @bio_alloc_bioset. Allows the caller * to ask for a number of bytes to be allocated in front of the bio. * Front pad allocation is useful for embedding the bio inside * another structure, to avoid allocating extra data to go with the bio. * Note that the bio must be embedded at the END of that structure always, * or things will break badly. * If %BIOSET_NEED_BVECS is set in @flags, a separate pool will be allocated * for allocating iovecs. This pool is not needed e.g. for bio_init_clone(). * If %BIOSET_NEED_RESCUER is set, a workqueue is created which can be used * to dispatch queued requests when the mempool runs out of space. * */int bioset_init(struct bio_set *bs, unsigned int pool_size, unsigned int front_pad, int flags){ bs->front_pad = front_pad; if (flags & BIOSET_NEED_BVECS) bs->back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec); else bs->back_pad = 0;
spin_lock_init(&bs->rescue_lock); bio_list_init(&bs->rescue_list); INIT_WORK(&bs->rescue_work, bio_alloc_rescue);
bs->bio_slab = bio_find_or_create_slab(bs); if (!bs->bio_slab) return -ENOMEM;
if (mempool_init_slab_pool(&bs->bio_pool, pool_size, bs->bio_slab)) goto bad;
if ((flags & BIOSET_NEED_BVECS) && biovec_init_pool(&bs->bvec_pool, pool_size)) goto bad;
if (flags & BIOSET_NEED_RESCUER) { bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM | WQ_PERCPU, 0); if (!bs->rescue_workqueue) goto bad; } if (flags & BIOSET_PERCPU_CACHE) { bs->cache = alloc_percpu(struct bio_alloc_cache); if (!bs->cache) goto bad; cpuhp_state_add_instance_nocalls(CPUHP_BIO_DEAD, &bs->cpuhp_dead); }
return 0;bad: bioset_exit(bs); return -ENOMEM;}EXPORT_SYMBOL(bioset_init);
static int __init init_bio(void){ int i;
BUILD_BUG_ON(BIO_FLAG_LAST > 8 * sizeof_field(struct bio, bi_flags));
for (i = 0; i < ARRAY_SIZE(bvec_slabs); i++) { struct biovec_slab *bvs = bvec_slabs + i;
bvs->slab = kmem_cache_create(bvs->name, bvs->nr_vecs * sizeof(struct bio_vec), 0, SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL); }
cpuhp_setup_state_multi(CPUHP_BIO_DEAD, "block/bio:dead", NULL, bio_cpu_dead);
if (bioset_init(&fs_bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS | BIOSET_PERCPU_CACHE)) panic("bio: can't allocate bios\n");
return 0;}subsys_initcall(init_bio);